Pause rolling-upgrade process of tidb statefulset

.gitignore

@@ -9,6 +9,7 @@ tests/images/fault-trigger/bin/
 tests/images/e2e/tidb-cluster/
 tests/images/e2e/tidb-backup/
 tests/images/e2e/tidb-operator/
+tests/images/e2e/manifests/
 *.tar
 tmp/
 data/

Makefile

@@ -28,7 +28,7 @@ docker-push: docker
 docker: build
 	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator
 
-build: controller-manager scheduler discovery
+build: controller-manager scheduler discovery admission-controller
 
 controller-manager:
 	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go
@@ -39,6 +39,9 @@ scheduler:
 discovery:
 	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-discovery cmd/discovery/main.go
 
+admission-controller:
+	$(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-admission-controller cmd/admission-controller/main.go
+
 e2e-setup:
 	# ginkgo doesn't work with retool for Go 1.11
 	@GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/[email protected]
@@ -50,9 +53,11 @@ e2e-docker: e2e-build
 	[ -d tests/images/e2e/tidb-operator ] && rm -r tests/images/e2e/tidb-operator || true
 	[ -d tests/images/e2e/tidb-cluster ] && rm -r tests/images/e2e/tidb-cluster || true
 	[ -d tests/images/e2e/tidb-backup ] && rm -r tests/images/e2e/tidb-backup || true
+	[ -d tests/images/e2e/manifests ] && rm -r tests/images/e2e/manifests || true
 	cp -r charts/tidb-operator tests/images/e2e
 	cp -r charts/tidb-cluster tests/images/e2e
 	cp -r charts/tidb-backup tests/images/e2e
+	cp -r manifests tests/images/e2e
 	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" tests/images/e2e
 
 e2e-build: e2e-setup

cmd/admission-controller/main.go

@@ -0,0 +1,96 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/golang/glog"
+	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
+	"github.com/pingcap/tidb-operator/pkg/webhook"
+	"github.com/pingcap/tidb-operator/version"
+	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+var (
+	printVersion bool
+	certFile     string
+	keyFile      string
+)
+
+func init() {
+	flag.BoolVar(&printVersion, "V", false, "Show version and quit")
+	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
+	flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.")
+	flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.")
+	flag.Parse()
+}
+
+func main() {
+
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if printVersion {
+		version.PrintVersionInfo()
+		os.Exit(0)
+	}
+	version.LogVersionInfo()
+
+	cfg, err := rest.InClusterConfig()
+	if err != nil {
+		glog.Fatalf("failed to get config: %v", err)
+	}
+
+	cli, err := versioned.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to create Clientset: %v", err)
+	}
+
+	kubeCli, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
+	}
+
+	webhookServer := webhook.NewWebHookServer(kubeCli, cli, certFile, keyFile)
+
+	sigs := make(chan os.Signal, 1)
+	done := make(chan bool, 1)
+
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		<-sigs
+
+		// Graceful shutdown the server
+		if err := webhookServer.Shutdown(); err != nil {
+			glog.Errorf("fail to shutdown server %v", err)
+		}
+
+		done <- true
+	}()
+
+	if err := webhookServer.Run(); err != nil {
+		glog.Errorf("stop http server %v", err)
+	}
+
+	<-done
+
+	glog.Infof("webhook server terminate safely.")
+}

images/tidb-operator/Dockerfile

@@ -4,3 +4,4 @@ RUN apk add tzdata --no-cache
 ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager
 ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler
 ADD bin/tidb-discovery /usr/local/bin/tidb-discovery
+ADD bin/tidb-admission-controller /usr/local/bin/tidb-admission-controller

manifests/create-cert.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    cat <<EOF
+Generate certificate suitable for use with an webhook service.
+
+This script uses k8s' CertificateSigningRequest API to a generate a
+certificate signed by k8s CA suitable for use with webhook
+services. This requires permissions to create and approve CSR. See
+https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster for
+detailed explantion and additional instructions.
+
+The server key/cert k8s CA cert are stored in a k8s secret.
+
+EOF
+    exit 1
+}
+
+namespace=default
+service=admission-controller-svc
+secret=admission-controller-certs
+
+if [ ! -x "$(command -v openssl)" ]; then
+    echo "openssl not found"
+    exit 1
+fi
+
+csrName=${service}.${namespace}
+tmpdir=$(mktemp -d)
+
+cat <<EOF >> ${tmpdir}/csr.conf
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = ${service}
+DNS.2 = ${service}.${namespace}
+DNS.3 = ${service}.${namespace}.svc
+EOF
+
+openssl genrsa -out ${tmpdir}/server-key.pem 2048
+openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf
+
+# clean-up any previously created CSR for our service. Ignore errors if not present.
+kubectl delete csr ${csrName} 2>/dev/null || true
+
+# create  server cert/key CSR and  send to k8s API
+cat <<EOF | kubectl create -f -
+apiVersion: certificates.k8s.io/v1beta1
+kind: CertificateSigningRequest
+metadata:
+  name: ${csrName}
+spec:
+  groups:
+  - system:authenticated
+  request: $(cat ${tmpdir}/server.csr | base64 | tr -d '\n')
+  usages:
+  - digital signature
+  - key encipherment
+  - server auth
+EOF
+
+# verify CSR has been created
+while true; do
+    kubectl get csr ${csrName}
+    if [ "$?" -eq 0 ]; then
+        break
+    fi
+done
+
+# approve and fetch the signed certificate
+kubectl certificate approve ${csrName}
+# verify certificate has been signed
+for x in $(seq 10); do
+    serverCert=$(kubectl get csr ${csrName} -o jsonpath='{.status.certificate}')
+    if [[ ${serverCert} != '' ]]; then
+        break
+    fi
+    sleep 1
+done
+if [[ ${serverCert} == '' ]]; then
+    echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2
+    exit 1
+fi
+
+echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem
+
+# create the secret with CA cert and server cert/key
+kubectl create secret generic ${secret} \
+        --from-file=key.pem=${tmpdir}/server-key.pem \
+        --from-file=cert.pem=${tmpdir}/server-cert.pem \
+        --dry-run -o yaml |
+    kubectl -n ${namespace} apply -f -

manifests/patch-ca.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+CURDIR=$(cd $(dirname ${BASH_SOURCE[0]}); pwd )
+
+CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n')
+echo $CA_BUNDLE
+sed -i "s/caBundle: .*$/caBundle: ${CA_BUNDLE}/g" $CURDIR/webhook.yaml

manifests/webhook.yaml

@@ -0,0 +1,109 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: admission-controller-cr
+  namespace: default
+  labels:
+    app: admission-webhook
+rules:
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get"]
+- apiGroups: ["pingcap.com"]
+  resources: ["tidbclusters"]
+  verbs: ["get"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admission-controller-sa
+  namespace: default
+  labels:
+    app: admission-controller
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: admission-controller-crb
+  namespace: default
+  labels:
+    app: admission-controller
+subjects:
+- kind: ServiceAccount
+  name: admission-controller-sa
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: admission-controller-cr
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: admission-controller-svc
+  namespace: default
+  labels:
+    app: admission-controller
+spec:
+  ports:
+  - port: 443
+    targetPort: 443
+  selector:
+    app: admission-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: admission-controller
+  namespace: default
+  labels:
+    app: admission-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: admission-controller
+  template:
+    metadata:
+      labels:
+        app: admission-controller
+    spec:
+      serviceAccountName: admission-controller-sa
+      containers:
+        - name: admission-controller
+          image: pingcap/tidb-operator:latest
+          imagePullPolicy: Always
+          command:
+            - /usr/local/bin/tidb-admission-controller
+            - -tlsCertFile=/etc/webhook/certs/cert.pem
+            - -tlsKeyFile=/etc/webhook/certs/key.pem
+            - -v=2
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: admission-controller-certs
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validation-admission-contorller-cfg
+  labels:
+    app: admission-controller
+webhooks:
+  - name: statefulset-admission-controller.pingcap.net
+    failurePolicy: Fail
+    clientConfig:
+      service:
+        name: admission-controller-svc
+        namespace: default
+        path: "/statefulsets"
+      caBundle: ${CA_BUNDLE}
+    rules:
+      - operations: [ "UPDATE" ]
+        apiGroups: [ "apps", "" ]
+        apiVersions: ["v1beta1"]
+        resources: ["statefulsets"]

pkg/label/label.go

@@ -50,6 +50,8 @@ const (
 	AnnPVCDeferDeleting = "tidb.pingcap.com/pvc-defer-deleting"
 	// AnnPVCPodScheduling is pod scheduling annotation key, it represents whether the pod is scheduling
 	AnnPVCPodScheduling = "tidb.pingcap.com/pod-scheduling"
+	// AnnTiDBPartition is pod annotation which TiDB pod chould upgrade to
+	AnnTiDBPartition string = "tidb.pingcap.com/tidb-partition"
 
 	// PDLabelVal is PD label value
 	PDLabelVal string = "pd"

pkg/tkctl/cmd/cmd.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/tkctl/cmd/get"
 	"github.com/pingcap/tidb-operator/pkg/tkctl/cmd/info"
 	"github.com/pingcap/tidb-operator/pkg/tkctl/cmd/list"
+	"github.com/pingcap/tidb-operator/pkg/tkctl/cmd/upinfo"
 	"github.com/pingcap/tidb-operator/pkg/tkctl/cmd/use"
 	"github.com/pingcap/tidb-operator/pkg/tkctl/config"
 	"github.com/spf13/cobra"
@@ -69,6 +70,7 @@ func NewTkcCommand(streams genericclioptions.IOStreams) *cobra.Command {
 				info.NewCmdInfo(tkcContext, streams),
 				use.NewCmdUse(tkcContext, streams),
 				version.NewCmdVersion(tkcContext, streams.Out),
+				upinfo.NewCmdUpInfo(tkcContext, streams),
 			},
 		},
 		{