diff --git a/.dockerignore b/.dockerignore index 243f81a5..a9863ed3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,7 @@ !**/*.go !**/*.mod !**/*.sum +.git +bin +testbin +vendor \ No newline at end of file diff --git a/Makefile b/Makefile index 54e8d613..5442965c 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,13 @@ # Image URL to use all building/pushing image targets -REGISTRY ?= quay.io FROM_VERSION ?=v2.2.1 CHART_VERSION ?=2.2.1 CHART_TOVERSION ?=2.3.0 TO_VERSION ?=v2.3.0 IMGPREFIX ?=radondb/ +MYSQL_IMAGE_57 ?=5.7.39 +MYSQL_IMAGE_80 ?=8.0.26 +MYSQL_IMAGE_57_TAG ?=$(IMGPREFIX)percona-server:$(MYSQL_IMAGE_57) +MYSQL_IMAGE_80_TAG ?=$(IMGPREFIX)percona-server:$(MYSQL_IMAGE_80) IMG ?= $(IMGPREFIX)mysql-operator:latest SIDECAR57_IMG ?= $(IMGPREFIX)mysql57-sidecar:latest SIDECAR80_IMG ?= $(IMGPREFIX)mysql80-sidecar:latest @@ -79,10 +82,13 @@ run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/manager/main.go docker-build: test ## Build docker image with the manager. - docker build --build-arg GO_PROXY=${GO_PORXY} -t ${IMG} . - docker build -f Dockerfile.sidecar --build-arg GO_PROXY=${GO_PORXY} -t ${SIDECAR57_IMG} . - docker build -f build/xenon/Dockerfile --build-arg GO_PROXY=${GO_PORXY} -t ${XENON_IMG} . - docker build --build-arg XTRABACKUP_PKG=percona-xtrabackup-80 --build-arg GO_PROXY=${GO_PORXY} -f Dockerfile.sidecar -t ${SIDECAR80_IMG} . + docker buildx build --build-arg GO_PROXY=${GO_PORXY} -t ${IMG} . + docker buildx build -f Dockerfile.sidecar --build-arg GO_PROXY=${GO_PORXY} -t ${SIDECAR57_IMG} . + docker buildx build -f build/xenon/Dockerfile --build-arg GO_PROXY=${GO_PORXY} -t ${XENON_IMG} . + docker buildx build --build-arg XTRABACKUP_PKG=percona-xtrabackup-80 --build-arg GO_PROXY=${GO_PORXY} -f Dockerfile.sidecar -t ${SIDECAR80_IMG} . + docker buildx build --build-arg "MYSQL_IMAGE=${MYSQL_IMAGE_57}" --build-arg GO_PROXY=${GO_PORXY} -f build/mysql/Dockerfile -t ${MYSQL_IMAGE_57_TAG} . + docker buildx build --build-arg "MYSQL_IMAGE=${MYSQL_IMAGE_80}" --build-arg GO_PROXY=${GO_PORXY} -f build/mysql/Dockerfile -t ${MYSQL_IMAGE_80_TAG} . + docker-push: ## Push docker image with the manager. docker push ${IMG} docker push ${SIDECAR_IMG} diff --git a/api/v1alpha1/mysqlcluster_types.go b/api/v1alpha1/mysqlcluster_types.go index bca42a30..19d4d0ed 100644 --- a/api/v1alpha1/mysqlcluster_types.go +++ b/api/v1alpha1/mysqlcluster_types.go @@ -175,6 +175,13 @@ type MysqlOpts struct { // +optional // +kubebuilder:default:={limits: {cpu: "500m", memory: "1Gi"}, requests: {cpu: "100m", memory: "256Mi"}} Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // MaxLagSeconds configures the readiness probe of mysqld container + // if the replication lag is greater than MaxLagSeconds, the mysqld container will not be not healthy. + // +kubebuilder:default:=30 + // +kubebuilder:validation:Minimum=0 + // +optional + MaxLagSeconds int `json:"maxLagTime,omitempty"` } // XenonOpts defines the options of xenon container. diff --git a/build/mysql/Dockerfile b/build/mysql/Dockerfile new file mode 100644 index 00000000..c72624a5 --- /dev/null +++ b/build/mysql/Dockerfile @@ -0,0 +1,25 @@ +# Build the manager binary +ARG MYSQL_IMAGE +FROM golang:1.17.13 as builder +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +ARG GO_PROXY=off +RUN if [ "$GO_PROXY" = "on" ]; then \ + go env -w GOPROXY=https://goproxy.cn,direct; \ + fi +RUN go mod download +# Copy the go source +COPY ./ . +# Build +RUN CGO_ENABLED=0 go build -a -o mysqlchecker ./cmd/mysql/main.go +FROM percona/percona-server:${MYSQL_IMAGE} +WORKDIR / +COPY --from=builder /workspace/mysqlchecker /usr/bin/mysqlchecker +USER 65532:65532 +ENTRYPOINT ["/docker-entrypoint.sh"] + +USER mysql +EXPOSE 3306 +CMD ["mysqld"] diff --git a/build/mysql/percona-server-5.7/Dockerfile b/build/mysql/percona-server-5.7/Dockerfile deleted file mode 100644 index ebba9764..00000000 --- a/build/mysql/percona-server-5.7/Dockerfile +++ /dev/null @@ -1,88 +0,0 @@ -FROM redhat/ubi8-minimal - -LABEL org.opencontainers.image.authors="info@percona.com" - -RUN set -ex; \ - # shadow-utils are needed for user/group manipulation on UBI-based images - microdnf -y update; \ - microdnf -y install glibc-langpack-en \ - nss_wrapper \ - shadow-utils; \ - microdnf clean all; \ - groupadd -g 1001 mysql; \ - useradd -u 1001 -r -g 1001 -s /sbin/nologin \ - -c "Default Application User" mysql - -# check repository package signature in secure way -RUN set -ex; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 99DB70FAE1D7CE227FB6488205B555B38483C65D; \ - gpg --batch --export --armor 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A > ${GNUPGHOME}/RPM-GPG-KEY-Percona; \ - gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ - rpmkeys --import ${GNUPGHOME}/RPM-GPG-KEY-Percona ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ - microdnf install -y findutils; \ - curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \ - rpmkeys --checksig /tmp/percona-release.rpm; \ - rpm -i /tmp/percona-release.rpm; \ - rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \ - rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY; \ - #microdnf -y module disable mysql; \ - curl -Lf -o /tmp/numactl-libs.rpm http://vault.centos.org/centos/8/BaseOS/x86_64/os/Packages/numactl-libs-2.0.12-13.el8.x86_64.rpm; \ - rpmkeys --checksig /tmp/numactl-libs.rpm; \ - rpm -i /tmp/numactl-libs.rpm; \ - rm -rf /tmp/numactl-libs.rpm - -ENV PS_VERSION 5.7.34-37.1 -ENV OS_VER el8 -ENV FULL_PERCONA_VERSION "$PS_VERSION.$OS_VER" - -RUN set -ex; \ - rpm -e --nodeps tzdata; \ - microdnf -y install \ - tzdata \ - jemalloc \ - which \ - cracklib-dicts \ - policycoreutils; \ - \ - #repoquery -a --location \ - # selinux-policy \ - # | xargs curl -Lf -o /tmp/selinux-policy.rpm; \ - #rpm -iv /tmp/selinux-policy.rpm --nodeps; \ - #rm -rf /tmp/selinux-policy.rpm; \ - \ - microdnf -y install \ - Percona-Server-server-57-${FULL_PERCONA_VERSION} \ - Percona-Server-devel-57-${FULL_PERCONA_VERSION} \ - Percona-Server-tokudb-57-${FULL_PERCONA_VERSION} \ - Percona-Server-rocksdb-57-${FULL_PERCONA_VERSION}; \ - microdnf clean all; \ - rm -rf /var/cache/dnf /var/cache/yum /var/lib/mysql - -# purge and re-create /var/lib/mysql with appropriate ownership -RUN set -ex; \ - /usr/bin/install -m 0775 -o mysql -g root -d /var/lib/mysql /var/run/mysqld /docker-entrypoint-initdb.d; \ -# comment out a few problematic configuration values - find /etc/percona-server.cnf /etc/percona-server.conf.d /etc/my.cnf.d -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log|user)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log|user)/#&/'; \ -# don't reverse lookup hostnames, they are usually another container - printf '[mysqld]\nskip-host-cache\nskip-name-resolve\n' > /etc/my.cnf.d/docker.cnf; \ -# TokuDB modifications - /usr/bin/install -m 0664 -o mysql -g root /dev/null /etc/sysconfig/mysql; \ - echo "LD_PRELOAD=/usr/lib64/libjemalloc.so.1" >> /etc/sysconfig/mysql; \ - echo "THP_SETTING=never" >> /etc/sysconfig/mysql; \ -# keep backward compatibility with debian images - ln -s /etc/my.cnf.d /etc/mysql; \ -# allow to change config files - chown -R mysql:root /etc/percona-server.cnf /etc/percona-server.conf.d /etc/my.cnf.d; \ - chmod -R ug+rwX /etc/percona-server.cnf /etc/percona-server.conf.d /etc/my.cnf.d - -VOLUME ["/var/lib/mysql", "/var/log/mysql"] - -COPY ps-entry.sh /docker-entrypoint.sh -ENTRYPOINT ["/docker-entrypoint.sh"] - -USER mysql -EXPOSE 3306 -CMD ["mysqld"] diff --git a/build/mysql/percona-server-5.7/ps-entry.sh b/build/mysql/percona-server-5.7/ps-entry.sh deleted file mode 100644 index 4be247d6..00000000 --- a/build/mysql/percona-server-5.7/ps-entry.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/bash -set -eo pipefail -shopt -s nullglob - -# if command starts with an option, prepend mysqld -if [ "${1:0:1}" = '-' ]; then - set -- mysqld "$@" -fi - -# skip setup if they want an option that stops mysqld -wantHelp= -for arg; do - case "$arg" in - -'?'|--help|--print-defaults|-V|--version) - wantHelp=1 - break - ;; - esac -done - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then - echo >&2 "error: both $var and $fileVar are set (but are exclusive)" - exit 1 - fi - local val="$def" - if [ "${!var:-}" ]; then - val="${!var}" - elif [ "${!fileVar:-}" ]; then - val="$(< "${!fileVar}")" - fi - export "$var"="$val" - unset "$fileVar" -} - -# usage: process_init_file FILENAME MYSQLCOMMAND... -# ie: process_init_file foo.sh mysql -uroot -# (process a single initializer file, based on its extension. we define this -# function here, so that initializer scripts (*.sh) can use the same logic, -# potentially recursively, or override the logic used in subsequent calls) -process_init_file() { - local f="$1"; shift - local mysql=( "$@" ) - - case "$f" in - *.sh) echo "$0: running $f"; . "$f" ;; - *.sql) echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo -} - -_check_config() { - toRun=( "$@" --verbose --help ) - if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then - cat >&2 <<-EOM - - ERROR: mysqld failed while attempting to check config - command was: "${toRun[*]}" - - $errors - EOM - exit 1 - fi -} - -# Fetch value from server config -# We use mysqld --verbose --help instead of my_print_defaults because the -# latter only show values present in config files, and not server defaults -_get_config() { - local conf="$1"; shift - "$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null \ - | awk '$1 == "'"$conf"'" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' - # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" -} - -if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then - # still need to check config, container may have started with --user - _check_config "$@" - - if [ -n "$INIT_TOKUDB" ]; then - export LD_PRELOAD=/usr/lib64/libjemalloc.so.1 - fi - # Get config - DATADIR="$(_get_config 'datadir' "$@")" - - if [ ! -d "$DATADIR/mysql" ]; then - file_env 'MYSQL_ROOT_PASSWORD' - if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then - echo >&2 'error: database is uninitialized and password option is not specified ' - echo >&2 ' You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD' - exit 1 - fi - - mkdir -p "$DATADIR" - - echo 'Initializing database' - "$@" --initialize-insecure --skip-ssl - echo 'Database initialized' - - if command -v mysql_ssl_rsa_setup > /dev/null && [ ! -e "$DATADIR/server-key.pem" ]; then - # https://github.com/mysql/mysql-server/blob/23032807537d8dd8ee4ec1c4d40f0633cd4e12f9/packaging/deb-in/extra/mysql-systemd-start#L81-L84 - echo 'Initializing certificates' - mysql_ssl_rsa_setup --datadir="$DATADIR" - echo 'Certificates initialized' - fi - - SOCKET="$(_get_config 'socket' "$@")" - "$@" --skip-networking --socket="${SOCKET}" & - pid="$!" - - mysql=( mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" --password="" ) - - for i in {120..0}; do - if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then - break - fi - echo 'MySQL init process in progress...' - sleep 1 - done - if [ "$i" = 0 ]; then - echo >&2 'MySQL init process failed.' - exit 1 - fi - - if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then - # sed is for https://bugs.mysql.com/bug.php?id=20545 - mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql - fi - - # install TokuDB engine - if [ -n "$INIT_TOKUDB" ]; then - ps-admin --docker --enable-tokudb -u root -p $MYSQL_ROOT_PASSWORD - fi - if [ -n "$INIT_ROCKSDB" ]; then - ps-admin --docker --enable-rocksdb -u root -p $MYSQL_ROOT_PASSWORD - fi - - if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then - MYSQL_ROOT_PASSWORD="$(pwmake 128)" - echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD" - fi - - rootCreate= - # default root to listen for connections from anywhere - file_env 'MYSQL_ROOT_HOST' '%' - if [ ! -z "$MYSQL_ROOT_HOST" -a "$MYSQL_ROOT_HOST" != 'localhost' ]; then - # no, we don't care if read finds a terminating character in this heredoc - # https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151 - read -r -d '' rootCreate <<-EOSQL || true - CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; - GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ; - EOSQL - fi - - "${mysql[@]}" <<-EOSQL - -- What's done in this file shouldn't be replicated - -- or products like mysql-fabric won't work - SET @@SESSION.SQL_LOG_BIN=0; - - DELETE FROM mysql.user WHERE user NOT IN ('mysql.sys', 'mysqlxsys', 'root') OR host NOT IN ('localhost') ; - SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; - GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ; - ${rootCreate} - DROP DATABASE IF EXISTS test ; - FLUSH PRIVILEGES ; - EOSQL - - if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then - mysql+=( -p"${MYSQL_ROOT_PASSWORD}" ) - fi - - file_env 'MYSQL_DATABASE' - if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}" - mysql+=( "$MYSQL_DATABASE" ) - fi - - file_env 'MYSQL_USER' - file_env 'MYSQL_PASSWORD' - if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}" - - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}" - fi - - echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}" - fi - - echo - ls /docker-entrypoint-initdb.d/ > /dev/null - for f in /docker-entrypoint-initdb.d/*; do - process_init_file "$f" "${mysql[@]}" - done - - if [ ! -z "$MYSQL_ONETIME_PASSWORD" ]; then - "${mysql[@]}" <<-EOSQL - ALTER USER 'root'@'%' PASSWORD EXPIRE; - EOSQL - fi - if ! kill -s TERM "$pid" || ! wait "$pid"; then - echo >&2 'MySQL init process failed.' - exit 1 - fi - - echo - echo 'MySQL init process done. Ready for start up.' - echo - fi - - # exit when MYSQL_INIT_ONLY environment variable is set to avoid starting mysqld - if [ ! -z "$MYSQL_INIT_ONLY" ]; then - echo 'Initialization complete, now exiting!' - exit 0 - fi -fi - -exec "$@" diff --git a/build/mysql/percona-server-8.0/Dockerfile b/build/mysql/percona-server-8.0/Dockerfile deleted file mode 100644 index a652c21b..00000000 --- a/build/mysql/percona-server-8.0/Dockerfile +++ /dev/null @@ -1,83 +0,0 @@ -# This Dockerfile should be used for docker official repo - -# https://github.com/docker-library/official-images: -# No official images can be derived from, or depend on, non-official images -# with the following notable exceptions... -FROM oraclelinux:8 - -LABEL org.opencontainers.image.authors="info@percona.com" - -# It is intentionally used another UID, to have backward compatibility with -# the previous image versions published on Docker Hub -RUN set -ex; \ - groupdel input; \ - userdel systemd-coredump; \ - groupadd -g 1001 mysql; \ - useradd -u 1001 -r -g 1001 -s /sbin/nologin \ - -c "Default Application User" mysql - -# check repository package signature in secure way -RUN set -ex; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 99DB70FAE1D7CE227FB6488205B555B38483C65D; \ - gpg --batch --export --armor 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A > ${GNUPGHOME}/RPM-GPG-KEY-Percona; \ - gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ - rpmkeys --import ${GNUPGHOME}/RPM-GPG-KEY-Percona ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ - curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \ - rpmkeys --checksig /tmp/percona-release.rpm; \ - rpm -i /tmp/percona-release.rpm; \ - rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \ - rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY; \ - dnf -y module disable mysql; \ - percona-release disable all; \ - percona-release enable ps-80 release - -ENV PS_VERSION 8.0.25-15.1 -ENV OS_VER el8 -ENV FULL_PERCONA_VERSION "$PS_VERSION.$OS_VER" - -RUN set -ex; \ - rpm -e --nodeps tzdata; \ - dnf -y install \ - hostname \ - tzdata \ - jemalloc \ - which \ - cracklib-dicts \ - tar \ - policycoreutils; \ - \ - dnf -y install \ - percona-server-server-${FULL_PERCONA_VERSION} \ - #percona-server-tokudb-${FULL_PERCONA_VERSION} \ - percona-server-devel-${FULL_PERCONA_VERSION} \ - percona-server-rocksdb-${FULL_PERCONA_VERSION}; \ - dnf clean all; \ - rm -rf /var/cache/dnf /var/cache/yum /var/lib/mysql - -# purge and re-create /var/lib/mysql with appropriate ownership -RUN set -ex; \ - /usr/bin/install -m 0775 -o mysql -g root -d /var/lib/mysql /var/run/mysqld /docker-entrypoint-initdb.d; \ -# comment out a few problematic configuration values - find /etc/my.cnf /etc/my.cnf.d -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log|user)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log|user)/#&/'; \ -# don't reverse lookup hostnames, they are usually another container - echo '!includedir /etc/my.cnf.d' >> /etc/my.cnf; \ - printf '[mysqld]\nskip-host-cache\nskip-name-resolve\n' > /etc/my.cnf.d/docker.cnf; \ -# TokuDB modifications - /usr/bin/install -m 0664 -o mysql -g root /dev/null /etc/sysconfig/mysql; \ - echo "LD_PRELOAD=/usr/lib64/libjemalloc.so.1" >> /etc/sysconfig/mysql; \ - echo "THP_SETTING=never" >> /etc/sysconfig/mysql; \ -# allow to change config files - chown -R mysql:root /etc/my.cnf /etc/my.cnf.d; \ - chmod -R ug+rwX /etc/my.cnf /etc/my.cnf.d - -VOLUME ["/var/lib/mysql", "/var/log/mysql"] - -COPY ps-entry.sh /docker-entrypoint.sh -ENTRYPOINT ["/docker-entrypoint.sh"] - -USER mysql -EXPOSE 3306 33060 -CMD ["mysqld"] diff --git a/build/mysql/percona-server-8.0/ps-entry.sh b/build/mysql/percona-server-8.0/ps-entry.sh deleted file mode 100644 index e356917d..00000000 --- a/build/mysql/percona-server-8.0/ps-entry.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/bin/bash -set -eo pipefail -shopt -s nullglob - -# if command starts with an option, prepend mysqld -if [ "${1:0:1}" = '-' ]; then - set -- mysqld "$@" -fi - -# skip setup if they want an option that stops mysqld -wantHelp= -for arg; do - case "$arg" in - -'?'|--help|--print-defaults|-V|--version) - wantHelp=1 - break - ;; - esac -done - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then - echo >&2 "error: both $var and $fileVar are set (but are exclusive)" - exit 1 - fi - local val="$def" - if [ "${!var:-}" ]; then - val="${!var}" - elif [ "${!fileVar:-}" ]; then - val="$(< "${!fileVar}")" - fi - export "$var"="$val" - unset "$fileVar" -} - -# usage: process_init_file FILENAME MYSQLCOMMAND... -# ie: process_init_file foo.sh mysql -uroot -# (process a single initializer file, based on its extension. we define this -# function here, so that initializer scripts (*.sh) can use the same logic, -# potentially recursively, or override the logic used in subsequent calls) -process_init_file() { - local f="$1"; shift - local mysql=( "$@" ) - - case "$f" in - *.sh) echo "$0: running $f"; . "$f" ;; - *.sql) echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo -} - -_check_config() { - toRun=( "$@" --verbose --help ) - if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then - cat >&2 <<-EOM - - ERROR: mysqld failed while attempting to check config - command was: "${toRun[*]}" - - $errors - EOM - exit 1 - fi -} - -# Fetch value from server config -# We use mysqld --verbose --help instead of my_print_defaults because the -# latter only show values present in config files, and not server defaults -_get_config() { - local conf="$1"; shift - "$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null \ - | awk '$1 == "'"$conf"'" && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' - # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" -} - -if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then - # still need to check config, container may have started with --user - _check_config "$@" - - if [ -n "$INIT_TOKUDB" ]; then - export LD_PRELOAD=/usr/lib64/libjemalloc.so.1 - fi - # Get config - DATADIR="$(_get_config 'datadir' "$@")" - - if [ ! -d "$DATADIR/mysql" ]; then - file_env 'MYSQL_ROOT_PASSWORD' - if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then - echo >&2 'error: database is uninitialized and password option is not specified ' - echo >&2 ' You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD' - exit 1 - fi - - mkdir -p "$DATADIR" - - echo 'Initializing database' - "$@" --initialize-insecure - echo 'Database initialized' - - if command -v mysql_ssl_rsa_setup > /dev/null && [ ! -e "$DATADIR/server-key.pem" ]; then - # https://github.com/mysql/mysql-server/blob/23032807537d8dd8ee4ec1c4d40f0633cd4e12f9/packaging/deb-in/extra/mysql-systemd-start#L81-L84 - echo 'Initializing certificates' - mysql_ssl_rsa_setup --datadir="$DATADIR" - echo 'Certificates initialized' - fi - - SOCKET="$(_get_config 'socket' "$@")" - "$@" --skip-networking --socket="${SOCKET}" & - pid="$!" - - mysql=( mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" --password="" ) - - for i in {120..0}; do - if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then - break - fi - echo 'MySQL init process in progress...' - sleep 1 - done - if [ "$i" = 0 ]; then - echo >&2 'MySQL init process failed.' - exit 1 - fi - - if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then - ( - echo "SET @@SESSION.SQL_LOG_BIN = off;" - # sed is for https://bugs.mysql.com/bug.php?id=20545 - mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' - ) | "${mysql[@]}" mysql - fi - - # install TokuDB engine - if [ -n "$INIT_TOKUDB" ]; then - ps-admin --docker --enable-tokudb -u root -p $MYSQL_ROOT_PASSWORD - fi - if [ -n "$INIT_ROCKSDB" ]; then - ps-admin --enable-rocksdb -u root -p $MYSQL_ROOT_PASSWORD - fi - - if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then - MYSQL_ROOT_PASSWORD="$(pwmake 128)" - echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD" - fi - - rootCreate= - # default root to listen for connections from anywhere - file_env 'MYSQL_ROOT_HOST' '%' - if [ ! -z "$MYSQL_ROOT_HOST" -a "$MYSQL_ROOT_HOST" != 'localhost' ]; then - # no, we don't care if read finds a terminating character in this heredoc - # https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151 - read -r -d '' rootCreate <<-EOSQL || true - CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; - GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ; - EOSQL - fi - - "${mysql[@]}" <<-EOSQL - -- What's done in this file shouldn't be replicated - -- or products like mysql-fabric won't work - SET @@SESSION.SQL_LOG_BIN=0; - - DELETE FROM mysql.user WHERE user NOT IN ('mysql.sys', 'mysqlxsys', 'mysql.infoschema', 'mysql.session', 'root') OR host NOT IN ('localhost') ; - ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; - GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ; - ${rootCreate} - DROP DATABASE IF EXISTS test ; - FLUSH PRIVILEGES ; - EOSQL - - if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then - mysql+=( -p"${MYSQL_ROOT_PASSWORD}" ) - fi - - file_env 'MYSQL_DATABASE' - if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}" - mysql+=( "$MYSQL_DATABASE" ) - fi - - file_env 'MYSQL_USER' - file_env 'MYSQL_PASSWORD' - if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}" - - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}" - fi - - echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}" - fi - - echo - ls /docker-entrypoint-initdb.d/ > /dev/null - for f in /docker-entrypoint-initdb.d/*; do - process_init_file "$f" "${mysql[@]}" - done - - if [ ! -z "$MYSQL_ONETIME_PASSWORD" ]; then - "${mysql[@]}" <<-EOSQL - ALTER USER 'root'@'%' PASSWORD EXPIRE; - EOSQL - fi - if ! kill -s TERM "$pid" || ! wait "$pid"; then - echo >&2 'MySQL init process failed.' - exit 1 - fi - - echo - echo 'MySQL init process done. Ready for start up.' - echo - fi - - # exit when MYSQL_INIT_ONLY environment variable is set to avoid starting mysqld - if [ ! -z "$MYSQL_INIT_ONLY" ]; then - echo 'Initialization complete, now exiting!' - exit 0 - fi -fi - -exec "$@" diff --git a/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml b/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml index d31bc46e..0f76cc53 100644 --- a/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml +++ b/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml @@ -169,6 +169,13 @@ spec: default: false description: InitTokuDB represents if install tokudb engine. type: boolean + maxLagTime: + default: 30 + description: MaxLagSeconds configures the readiness probe of mysqld + container if the replication lag is greater than MaxLagSeconds, + the mysqld container will not be not healthy. + minimum: 0 + type: integer mysqlConf: additionalProperties: type: string diff --git a/cmd/mysql/main.go b/cmd/mysql/main.go new file mode 100644 index 00000000..dc4d9a70 --- /dev/null +++ b/cmd/mysql/main.go @@ -0,0 +1,357 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "github.com/go-ini/ini" + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + "github.com/radondb/radondb-mysql-kubernetes/utils" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +var ( + clientConfDir = "/etc/my.cnf.d/client.conf" + connectionMaxIdleTime = 30 * time.Second + connectionTimeout = 30 * time.Second + raftStatusCmd = "xenoncli raft status" +) + +type RaftStatus struct { + State string `json:"state"` + Leader string `json:"leader"` + Nodes []string `json:"nodes"` +} + +type SlaveStatus struct { + LastIOErrno int `db:"Last_IO_Errno"` + LastIOError string `db:"Last_IO_Error"` + LastSQLErrno int `db:"Last_SQL_Errno"` + LastSQLError string `db:"Last_SQL_Error"` + MasterHost string `db:"Master_Host"` + RetrievedGtidSet string `db:"Retrieved_Gtid_Set"` + ExecutedGtidSet string `db:"Executed_Gtid_Set"` + SlaveIORunning string `db:"Slave_IO_Running"` + SlaveSQLRunning string `db:"Slave_SQL_Running"` + SlaveIOState string `db:"Slave_IO_State"` + MasterUser string `db:"Master_User"` + MasterPort int `db:"Master_Port"` + ConnectRetry int `db:"Connect_Retry"` + MasterLogFile string `db:"Master_Log_File"` + ReadMasterLogPos int `db:"Read_Master_Log_Pos"` + RelayLogFile string `db:"Relay_Log_File"` + RelayLogPos int `db:"Relay_Log_Pos"` + RelayMasterLogFile string `db:"Relay_Master_Log_File"` + ReplicateDoDB string `db:"Replicate_Do_DB"` + ReplicateIgnoreDB string `db:"Replicate_Ignore_DB"` + ReplicateDoTable string `db:"Replicate_Do_Table"` + ReplicateIgnoreTable string `db:"Replicate_Ignore_Table"` + ReplicateWildDoTable string `db:"Replicate_Wild_Do_Table"` + ReplicateWildIgnoreTable string `db:"Replicate_Wild_Ignore_Table"` + LastErrno int `db:"Last_Errno"` + LastError string `db:"Last_Error"` + SkipCounter int `db:"Skip_Counter"` + ExecMasterLogPos int `db:"Exec_Master_Log_Pos"` + RelayLogSpace int `db:"Relay_Log_Space"` + UntilCondition string `db:"Until_Condition"` + UntilLogFile string `db:"Until_Log_File"` + UntilLogPos int `db:"Until_Log_Pos"` + MasterSSLAllowed string `db:"Master_SSL_Allowed"` + MasterSSLCAFile string `db:"Master_SSL_CA_File"` + MasterSSLCAPath string `db:"Master_SSL_CA_Path"` + MasterSSLCert string `db:"Master_SSL_Cert"` + MasterSSLCipher string `db:"Master_SSL_Cipher"` + MasterSSLKey string `db:"Master_SSL_Key"` + SecondsBehindMaster sql.NullInt64 `db:"Seconds_Behind_Master"` + MasterSSLVerifyServerCert string `db:"Master_SSL_Verify_Server_Cert"` + ReplicateIgnoreServerIds string `db:"Replicate_Ignore_Server_Ids"` + MasterServerID int `db:"Master_Server_Id"` + MasterUUID string `db:"Master_UUID"` + MasterInfoFile string `db:"Master_Info_File"` + SQLDelay int `db:"SQL_Delay"` + SQLRemainingDelay sql.NullInt64 `db:"SQL_Remaining_Delay"` + SlaveSQLRunningState string `db:"Slave_SQL_Running_State"` + MasterRetryCount int `db:"Master_Retry_Count"` + MasterBind string `db:"Master_Bind"` + LastIOErrorTimestamp string `db:"Last_IO_Error_Timestamp"` + LastSQLErrorTimestamp string `db:"Last_SQL_Error_Timestamp"` + MasterSSLCrl string `db:"Master_SSL_Crl"` + MasterSSLCrlpath string `db:"Master_SSL_Crlpath"` + AutoPosition string `db:"Auto_Position"` + ReplicateRewriteDB string `db:"Replicate_Rewrite_DB"` + ChannelName string `db:"Channel_Name"` + MasterTLSVersion string `db:"Master_TLS_Version"` + Masterpublickeypath string `db:"Master_public_key_path"` + Getmasterpublickey string `db:"Get_master_public_key"` + NetworkNamespace string `db:"Network_Namespace"` +} + +type Agent struct { + conf MySQLConfig + db *sqlx.DB + maxDelay time.Duration + ksClient *kubernetes.Clientset + podName string + nameSpace string +} + +type MySQLConfig struct { + Host string + Port int + User string + Password string +} +type GTID struct { + ID string `json:"id"` + Raft string `json:"raft"` + Mysql string `json:"mysql"` + ExecutedGTIDSet string `json:"executed-gtid-set"` + RetrievedGTIDSet string `json:"retrieved-gtid-set"` +} + +func New() *Agent { + debugFlag, _ := strconv.ParseBool(os.Getenv("RADONDB_DEBUG")) + if debugFlag { + log.SetLevel(log.DebugLevel) + } + conf := getMySQLclientConf() + db, err := getMySQLConn(conf) + if err != nil { + log.Fatalf("get mysql connection failed: %v", err) + } + maxDelay, _ := strconv.Atoi(os.Getenv("MAX_DELAY")) + ksCgent, err := utils.GetClientSet() + if err != nil { + log.Fatalf("get kubernetes clientset failed: %v", err) + } + podName := os.Getenv("POD_NAME") + nameSpace := os.Getenv("NAMESPACE") + + return &Agent{ + conf: *conf, + db: db, + maxDelay: time.Duration(maxDelay) * time.Second, + ksClient: ksCgent, + podName: podName, + nameSpace: nameSpace, + } +} + +func main() { + if len(os.Args) < 2 { + log.Fatalf("Usage: %s leaderStart|leaderStop|liveness|readiness|postStart|preStop", os.Args[0]) + } + agent := New() + defer agent.CloseDB() + switch os.Args[1] { + case "liveness": + if err := agent.liveness(); err != nil { + log.Fatalf("liveness failed: %s", err.Error()) + } + case "readiness": + if err := agent.readiness(); err != nil { + log.Fatalf("readiness failed: %s", err.Error()) + } + case "postStart": + if err := agent.postStart(); err != nil { + log.Fatalf("postStart failed: %s", err.Error()) + } + case "preStop": + if err := agent.preStop(); err != nil { + log.Fatalf("postStop failed: %s", err.Error()) + } + default: + log.Fatalf("Usage: %s leaderStart|leaderStop|liveness|readiness|postStart|preStop", os.Args[0]) + } +} +func (c *Agent) liveness() error { + // if sleep-forever is set, then skip + if utils.SleepFlag() { + return nil + } + // get mysql pid from pid file + if utils.IsMySQLRunning() { + return nil + } else { + return fmt.Errorf("mysql is not running") + } +} + +func (c *Agent) readiness() error { + // Check the instance works primary or not + rows, err := c.db.Query("select @@read_only") + if err != nil { + return err + } + defer rows.Close() + var readOnly bool + for rows.Next() { + if err := rows.Scan(&readOnly); err != nil { + return err + } + } + role, err := c.getRoleBylabel() + if err != nil { + return err + } + // slave check delay + switch role { + case "FOLLOWER": + { + // check if the cluster has leader + raftStatus, err := c.getRaftStatus() + if err != nil { + return err + } + if raftStatus.Leader == "" { + log.Warning("no leader found,skip readiness check") + return nil + } + status := &SlaveStatus{} + err1 := c.db.GetContext(context.Background(), status, `show slave status`) + if err1 != nil { + return err + } + + if status.SlaveIORunning != "Yes" || status.SlaveSQLRunning != "Yes" { + return fmt.Errorf("replication threads are stopped") + } + if status.LastError != "" { + return fmt.Errorf("slave has error: %s", status.LastError) + } + if status.SecondsBehindMaster.Int64 > int64(c.maxDelay.Seconds()) { + return fmt.Errorf("slave is too far behind master") + } + } + case "LEADER": + { + if !utils.ExistUpdateFile() && readOnly { + log.Errorf("am leader but read_only is on") + if err := c.setGlobalReadOnlyOff(); err != nil { + return err + } + } + + } + + } + return nil +} + +func (c *Agent) postStart() error { + return nil +} + +func (c *Agent) preStop() error { + return nil +} +func (c *Agent) CloseDB() error { + return c.db.Close() +} + +func getMySQLclientConf() *MySQLConfig { + // read config file + cfg, err := ini.Load(clientConfDir) + if err != nil { + log.Fatalf("Fail to read file: %v", err) + } + // read section + section, err := cfg.GetSection("client") + if err != nil { + log.Fatalf("Fail to get section: %v", err) + } + // read key + host := section.Key("host").String() + port, err := section.Key("port").Int() + if err != nil { + log.Fatalf("Fail to get port: %v", err) + } + password := section.Key("password").String() + user := section.Key("user").String() + return &MySQLConfig{ + Host: host, + Port: port, + User: user, + Password: password, + } + +} + +func getMySQLConn(conf *MySQLConfig) (*sqlx.DB, error) { + c := mysql.NewConfig() + c.User = conf.User + c.Passwd = conf.Password + c.Net = "tcp" + c.Addr = fmt.Sprintf("%s:%d", conf.Host, conf.Port) + c.Timeout = connectionTimeout + c.ReadTimeout = connectionTimeout + c.InterpolateParams = true + c.ParseTime = true + db, err := sqlx.Open("mysql", c.FormatDSN()) + if err != nil { + return nil, err + } + db.SetConnMaxIdleTime(connectionMaxIdleTime) + db.SetConnMaxLifetime(1 * time.Minute) + db.SetMaxIdleConns(1) + return db, nil +} + +func (c *Agent) getRoleBylabel() (string, error) { + podMeta, err := c.ksClient.CoreV1().Pods(c.nameSpace).Get(context.TODO(), c.podName, metav1.GetOptions{}) + if err != nil { + return "", err + } + if role, ok := podMeta.Labels["role"]; ok { + return role, nil + } + return "", fmt.Errorf("role label not found") +} + +func (c *Agent) setGlobalReadOnlyOff() error { + _, err := c.db.Exec("set global read_only=0") + if err != nil { + return fmt.Errorf("failed to disable super_read_only: %w", err) + } + return nil +} + +func (c *Agent) getRaftStatus() (*RaftStatus, error) { + config, err := utils.NewConfig() + if err != nil { + panic(err) + } + k, err := utils.NewForConfig(config) + if err != nil { + panic(err) + } + cfg := utils.RunRemoteCommandConfig{ + PodName: c.podName, + Namespace: c.nameSpace, + Container: "xenon", + } + + raftStatusCmd := []string{raftStatusCmd} + var output, stderr string + output, stderr, err = utils.RunRemoteCommand(k, cfg, raftStatusCmd) + log.Info("output=[" + output + "]") + log.Info("stderr=[" + stderr + "]") + if err != nil { + log.Fatal(err) + } + status := &RaftStatus{} + if err := json.Unmarshal([]byte(output), &status); err != nil { + log.Fatal(err) + return nil, err + } + return status, nil + +} diff --git a/cmd/xenon/main.go b/cmd/xenon/main.go index 09f20a71..4a6e17f0 100644 --- a/cmd/xenon/main.go +++ b/cmd/xenon/main.go @@ -6,7 +6,6 @@ import ( "database/sql" "encoding/json" "fmt" - "io" "os" "os/exec" "strconv" @@ -16,26 +15,11 @@ import ( _ "github.com/go-sql-driver/mysql" . "github.com/radondb/radondb-mysql-kubernetes/utils" log "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/remotecommand" ) -type KubeAPI struct { - Client *kubernetes.Clientset - Config *rest.Config -} - -type runRemoteCommandConfig struct { - container, namespace, podName string -} - const ( leaderStopCommand = "kill -9 $(pidof mysqld)" mysqlUser = "root" @@ -566,67 +550,6 @@ func patchPodLabel(n MySQLNode, patch string) error { return nil } -func (k *KubeAPI) Exec(namespace, pod, container string, stdin io.Reader, command []string) (string, string, error) { - var stdout, stderr bytes.Buffer - - var Scheme = runtime.NewScheme() - if err := corev1.AddToScheme(Scheme); err != nil { - log.Fatalf("failed to add to scheme: %v", err) - return "", "", err - } - var ParameterCodec = runtime.NewParameterCodec(Scheme) - - request := k.Client.CoreV1().RESTClient().Post(). - Resource("pods").SubResource("exec"). - Namespace(namespace).Name(pod). - VersionedParams(&corev1.PodExecOptions{ - Container: container, - Command: command, - Stdin: stdin != nil, - Stdout: true, - Stderr: true, - }, ParameterCodec) - - exec, err := remotecommand.NewSPDYExecutor(k.Config, "POST", request.URL()) - - if err == nil { - err = exec.Stream(remotecommand.StreamOptions{ - Stdin: stdin, - Stdout: &stdout, - Stderr: &stderr, - }) - } - - return stdout.String(), stderr.String(), err -} - -func runRemoteCommand(kubeapi *KubeAPI, cfg runRemoteCommandConfig, cmd []string) (string, string, error) { - bashCmd := []string{"bash"} - reader := strings.NewReader(strings.Join(cmd, " ")) - return kubeapi.Exec(cfg.namespace, cfg.podName, cfg.container, reader, bashCmd) -} - -func NewForConfig(config *rest.Config) (*KubeAPI, error) { - var api KubeAPI - var err error - - api.Config = config - api.Client, err = kubernetes.NewForConfig(api.Config) - - return &api, err -} - -func NewConfig() (*rest.Config, error) { - // The default loading rules try to read from the files specified in the - // environment or from the home directory. - loader := clientcmd.NewDefaultClientConfigLoadingRules() - - // The deferred loader tries an in-cluster config if the default loading - // rules produce no results. - return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - loader, &clientcmd.ConfigOverrides{}).ClientConfig() -} - func killMysqld() error { config, err := NewConfig() if err != nil { @@ -636,16 +559,16 @@ func killMysqld() error { if err != nil { panic(err) } - cfg := runRemoteCommandConfig{ - podName: podName, - namespace: ns, - container: "mysql", + cfg := RunRemoteCommandConfig{ + PodName: podName, + Namespace: ns, + Container: "mysql", } killMySQLCommand := []string{leaderStopCommand} log.Infof("killing mysql command: %s", leaderStopCommand) var output, stderr string - output, stderr, err = runRemoteCommand(k, cfg, killMySQLCommand) + output, stderr, err = RunRemoteCommand(k, cfg, killMySQLCommand) log.Info("output=[" + output + "]") log.Info("stderr=[" + stderr + "]") if err != nil { diff --git a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml index d31bc46e..0f76cc53 100644 --- a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml +++ b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml @@ -169,6 +169,13 @@ spec: default: false description: InitTokuDB represents if install tokudb engine. type: boolean + maxLagTime: + default: 30 + description: MaxLagSeconds configures the readiness probe of mysqld + container if the replication lag is greater than MaxLagSeconds, + the mysqld container will not be not healthy. + minimum: 0 + type: integer mysqlConf: additionalProperties: type: string diff --git a/go.mod b/go.mod index 10432983..6612d541 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/gruntwork-io/terratest v0.40.20 github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 github.com/imdario/mergo v0.3.12 + github.com/jmoiron/sqlx v1.3.5 github.com/onsi/ginkgo/v2 v2.1.3 github.com/onsi/gomega v1.19.0 github.com/presslabs/controller-util v0.4.3 diff --git a/go.sum b/go.sum index 4e8fc531..850ea170 100644 --- a/go.sum +++ b/go.sum @@ -575,6 +575,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -615,6 +617,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -632,6 +636,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= diff --git a/mysqlcluster/container/mysql.go b/mysqlcluster/container/mysql.go index 16929f02..54add740 100644 --- a/mysqlcluster/container/mysql.go +++ b/mysqlcluster/container/mysql.go @@ -55,16 +55,39 @@ func (c *mysql) getCommand() []string { // getEnvVars get the container env. func (c *mysql) getEnvVars() []corev1.EnvVar { - if c.Spec.MysqlOpts.InitTokuDB { - return []corev1.EnvVar{ - { - Name: "INIT_TOKUDB", - Value: "1", + envVars := []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, }, - } + }, + { + Name: "MAX_DELAY", + Value: fmt.Sprint(c.Spec.MysqlOpts.MaxLagSeconds), + }, } - return nil + if c.Spec.MysqlOpts.InitTokuDB { + envVars = append(envVars, corev1.EnvVar{ + Name: "INIT_TOKUDB", + Value: "1", + }) + } + + return envVars } // getLifecycle get the container lifecycle. @@ -97,9 +120,9 @@ func (c *mysql) getLivenessProbe() *corev1.Probe { kubectl exec -it sample-mysql-0 -c mysql -- sh -c 'touch /var/lib/mysql/sleep-forever' */ Command: []string{ - "sh", + "/usr/bin/bash", "-c", - "if [ -f '/var/lib/mysql/sleep-forever' ] ;then exit 0 ; fi; pgrep mysqld", + "mysqlchecker liveness", }, }, }, @@ -117,9 +140,9 @@ func (c *mysql) getReadinessProbe() *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ Command: []string{ - "sh", + "/usr/bin/bash", "-c", - fmt.Sprintf(`if [ -f '/var/lib/mysql/sleep-forever' ] ;then exit 0 ; fi; test $(mysql --defaults-file=%s -NB -e "SELECT 1") -eq 1`, utils.ConfClientPath), + "mysqlchecker readiness", }, }, }, diff --git a/mysqlcluster/container/mysql_test.go b/mysqlcluster/container/mysql_test.go index 9591eaf5..8ec17e51 100644 --- a/mysqlcluster/container/mysql_test.go +++ b/mysqlcluster/container/mysql_test.go @@ -65,11 +65,33 @@ func TestGetMysqlCommand(t *testing.T) { func TestGetMysqlEnvVar(t *testing.T) { // base env { - assert.Nil(t, mysqlCase.Env) + assert.NotNil(t, mysqlCase.Env) } // initTokuDB { volumeMounts := []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "MAX_DELAY", + Value: "0", + }, { Name: "INIT_TOKUDB", Value: "1", @@ -110,7 +132,7 @@ func TestGetMysqlLivenessProbe(t *testing.T) { livenessProbe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", "if [ -f '/var/lib/mysql/sleep-forever' ] ;then exit 0 ; fi; pgrep mysqld"}, + Command: []string{"/usr/bin/bash", "-c", "mysqlchecker liveness"}, }, }, InitialDelaySeconds: 30, @@ -126,7 +148,7 @@ func TestGetMysqlReadinessProbe(t *testing.T) { readinessProbe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", `if [ -f '/var/lib/mysql/sleep-forever' ] ;then exit 0 ; fi; test $(mysql --defaults-file=/etc/mysql/client.conf -NB -e "SELECT 1") -eq 1`}, + Command: []string{"/usr/bin/bash", "-c", "mysqlchecker readiness"}, }, }, InitialDelaySeconds: 10, diff --git a/mysqlcluster/syncer/status.go b/mysqlcluster/syncer/status.go index c60375a0..799d9917 100644 --- a/mysqlcluster/syncer/status.go +++ b/mysqlcluster/syncer/status.go @@ -305,14 +305,14 @@ func (s *StatusSyncer) updateNodeStatus(ctx context.Context, cli client.Client, s.log.V(1).Info("failed to check read only", "node", node.Name, "error", err) node.Message = err.Error() } - - if !utils.ExistUpdateFile() && - node.RaftStatus.Role == string(utils.Leader) && - isReadOnly != corev1.ConditionFalse { - s.log.V(1).Info("try to correct the leader writeable", "node", node.Name) - sqlRunner.QueryExec(internal.NewQuery("SET GLOBAL read_only=off")) - sqlRunner.QueryExec(internal.NewQuery("SET GLOBAL super_read_only=off")) - } + // move it to mysql readiness + // if !utils.ExistUpdateFile() && + // node.RaftStatus.Role == string(utils.Leader) && + // isReadOnly != corev1.ConditionFalse { + // s.log.V(1).Info("try to correct the leader writeable", "node", node.Name) + // sqlRunner.QueryExec(internal.NewQuery("SET GLOBAL read_only=off")) + // sqlRunner.QueryExec(internal.NewQuery("SET GLOBAL super_read_only=off")) + // } } // update apiv1alpha1.NodeConditionLagged. @@ -473,6 +473,14 @@ func (s *StatusSyncer) updatePodLabel(ctx context.Context, pod *corev1.Pod, node healthy = "no" node.RaftStatus.Role = string(utils.Unknown) } + // update healthy no if container is not ready. + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Name == utils.ContainerMysqlName && !containerStatus.Ready { + healthy = "no" + isPodLabelsUpdated = true + break + } + } if pod.Labels["healthy"] != healthy { pod.Labels["healthy"] = healthy diff --git a/utils/incluster.go b/utils/incluster.go index 2fc738e4..e3d39f06 100644 --- a/utils/incluster.go +++ b/utils/incluster.go @@ -1,18 +1,42 @@ package utils import ( + "bytes" "context" "encoding/json" "fmt" + "io" + "io/ioutil" "log" + "os" "os/exec" + "strconv" + "strings" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" ) +const ( + // MySQLPidFile is the path of mysql pid file. + MySQLPidFile = "/var/run/mysqld/mysqld.pid" + SleepFlagFile = "/var/lib/mysql/sleep-forever" +) + +type KubeAPI struct { + Client *kubernetes.Clientset + Config *rest.Config +} + +type RunRemoteCommandConfig struct { + Container, Namespace, PodName string +} type raftStatus struct { Leader string `json:"leader"` State string `json:"state"` @@ -79,3 +103,91 @@ func GetRaftStatus() *raftStatus { func GetRole() string { return GetRaftStatus().State } + +func SleepFlag() bool { + _, err := os.Stat(SleepFlagFile) + return !os.IsNotExist(err) +} + +func GetMySQLPid() (int, error) { + d, err := ioutil.ReadFile(MySQLPidFile) + if err != nil { + return -1, fmt.Errorf("failed to read mysql pid file: %v", err) + } + pid, err := strconv.Atoi(string(bytes.TrimSpace(d))) + if err != nil { + return -1, fmt.Errorf("error parsing pid from %s: %s", MySQLPidFile, err) + } + return pid, nil +} + +func IsMySQLRunning() bool { + pid, err := GetMySQLPid() + if err != nil { + return false + } + // check if the process is running + _, err = os.FindProcess(pid) + return err == nil +} + +func (k *KubeAPI) Exec(namespace, pod, container string, stdin io.Reader, command []string) (string, string, error) { + var stdout, stderr bytes.Buffer + + var Scheme = runtime.NewScheme() + if err := corev1.AddToScheme(Scheme); err != nil { + log.Fatalf("failed to add to scheme: %v", err) + return "", "", err + } + var ParameterCodec = runtime.NewParameterCodec(Scheme) + + request := k.Client.CoreV1().RESTClient().Post(). + Resource("pods").SubResource("exec"). + Namespace(namespace).Name(pod). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: command, + Stdin: stdin != nil, + Stdout: true, + Stderr: true, + }, ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(k.Config, "POST", request.URL()) + + if err == nil { + err = exec.Stream(remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &stdout, + Stderr: &stderr, + }) + } + + return stdout.String(), stderr.String(), err +} + +func RunRemoteCommand(kubeapi *KubeAPI, cfg RunRemoteCommandConfig, cmd []string) (string, string, error) { + bashCmd := []string{"bash"} + reader := strings.NewReader(strings.Join(cmd, " ")) + return kubeapi.Exec(cfg.Namespace, cfg.PodName, cfg.Container, reader, bashCmd) +} + +func NewForConfig(config *rest.Config) (*KubeAPI, error) { + var api KubeAPI + var err error + + api.Config = config + api.Client, err = kubernetes.NewForConfig(api.Config) + + return &api, err +} + +func NewConfig() (*rest.Config, error) { + // The default loading rules try to read from the files specified in the + // environment or from the home directory. + loader := clientcmd.NewDefaultClientConfigLoadingRules() + + // The deferred loader tries an in-cluster config if the default loading + // rules produce no results. + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + loader, &clientcmd.ConfigOverrides{}).ClientConfig() +} diff --git a/utils/incluster_test.go b/utils/incluster_test.go new file mode 100644 index 00000000..63b4ba74 --- /dev/null +++ b/utils/incluster_test.go @@ -0,0 +1,20 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSleepFlag(t *testing.T) { + sleep := SleepFlag() + assert.Equal(t, sleep, false) +} + +func TestIsMySQLRunning(t *testing.T) { + { + want := false + got := IsMySQLRunning() + assert.Equal(t, want, got) + } +}