diff --git a/.gitignore b/.gitignore index bab2459fdf6..c735f5de6da 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tests/images/fault-trigger/bin/ tests/images/e2e/tidb-cluster/ tests/images/e2e/tidb-backup/ tests/images/e2e/tidb-operator/ +tests/images/e2e/manifests/ *.tar tmp/ data/ diff --git a/Makefile b/Makefile index a985090f312..243797757ef 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager scheduler discovery +build: controller-manager scheduler discovery admission-controller controller-manager: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go @@ -39,6 +39,9 @@ scheduler: discovery: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-discovery cmd/discovery/main.go +admission-controller: + $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-admission-controller cmd/admission-controller/main.go + e2e-setup: # ginkgo doesn't work with retool for Go 1.11 @GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/ginkgo@v1.6.0 @@ -50,9 +53,11 @@ e2e-docker: e2e-build [ -d tests/images/e2e/tidb-operator ] && rm -r tests/images/e2e/tidb-operator || true [ -d tests/images/e2e/tidb-cluster ] && rm -r tests/images/e2e/tidb-cluster || true [ -d tests/images/e2e/tidb-backup ] && rm -r tests/images/e2e/tidb-backup || true + [ -d tests/images/e2e/manifests ] && rm -r tests/images/e2e/manifests || true cp -r charts/tidb-operator tests/images/e2e cp -r charts/tidb-cluster tests/images/e2e cp -r charts/tidb-backup tests/images/e2e + cp -r manifests tests/images/e2e docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" tests/images/e2e e2e-build: e2e-setup diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go new file mode 100644 index 00000000000..c9b16722403 --- /dev/null +++ b/cmd/admission-controller/main.go @@ -0,0 +1,96 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "os/signal" + "syscall" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/webhook" + "github.com/pingcap/tidb-operator/version" + "k8s.io/apiserver/pkg/util/logs" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + printVersion bool + certFile string + keyFile string +) + +func init() { + flag.BoolVar(&printVersion, "V", false, "Show version and quit") + flag.BoolVar(&printVersion, "version", false, "Show version and quit") + flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.") + flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.") + flag.Parse() +} + +func main() { + + logs.InitLogs() + defer logs.FlushLogs() + + if printVersion { + version.PrintVersionInfo() + os.Exit(0) + } + version.LogVersionInfo() + + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Fatalf("failed to get config: %v", err) + } + + cli, err := versioned.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to create Clientset: %v", err) + } + + kubeCli, err := kubernetes.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to get kubernetes Clientset: %v", err) + } + + webhookServer := webhook.NewWebHookServer(kubeCli, cli, certFile, keyFile) + + sigs := make(chan os.Signal, 1) + done := make(chan bool, 1) + + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigs + + // Graceful shutdown the server + if err := webhookServer.Shutdown(); err != nil { + glog.Errorf("fail to shutdown server %v", err) + } + + done <- true + }() + + if err := webhookServer.Run(); err != nil { + glog.Errorf("stop http server %v", err) + } + + <-done + + glog.Infof("webhook server terminate safely.") +} diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index 52679208071..2db8eeb21b9 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -4,3 +4,4 @@ RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery +ADD bin/tidb-admission-controller /usr/local/bin/tidb-admission-controller diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh new file mode 100755 index 00000000000..4e76ff8fa02 --- /dev/null +++ b/manifests/create-cert.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + +# clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + +# create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi + +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + +# create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=key.pem=${tmpdir}/server-key.pem \ + --from-file=cert.pem=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - diff --git a/manifests/patch-ca.sh b/manifests/patch-ca.sh new file mode 100755 index 00000000000..f222369e906 --- /dev/null +++ b/manifests/patch-ca.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +CURDIR=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) + +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +echo $CA_BUNDLE +sed -i "s/caBundle: .*$/caBundle: ${CA_BUNDLE}/g" $CURDIR/webhook.yaml diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml new file mode 100644 index 00000000000..489c14003bc --- /dev/null +++ b/manifests/webhook.yaml @@ -0,0 +1,109 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: admission-controller-cr + namespace: default + labels: + app: admission-webhook +rules: +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get"] +- apiGroups: ["pingcap.com"] + resources: ["tidbclusters"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-controller-sa + namespace: default + labels: + app: admission-controller +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: admission-controller-crb + namespace: default + labels: + app: admission-controller +subjects: +- kind: ServiceAccount + name: admission-controller-sa + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admission-controller-cr +--- +apiVersion: v1 +kind: Service +metadata: + name: admission-controller-svc + namespace: default + labels: + app: admission-controller +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: admission-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-controller + namespace: default + labels: + app: admission-controller +spec: + replicas: 1 + selector: + matchLabels: + app: admission-controller + template: + metadata: + labels: + app: admission-controller + spec: + serviceAccountName: admission-controller-sa + containers: + - name: admission-controller + image: pingcap/tidb-operator:latest + imagePullPolicy: Always + command: + - /usr/local/bin/tidb-admission-controller + - -tlsCertFile=/etc/webhook/certs/cert.pem + - -tlsKeyFile=/etc/webhook/certs/key.pem + - -v=2 + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: admission-controller-certs +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation-admission-contorller-cfg + labels: + app: admission-controller +webhooks: + - name: statefulset-admission-controller.pingcap.net + failurePolicy: Fail + clientConfig: + service: + name: admission-controller-svc + namespace: default + path: "/statefulsets" + caBundle: ${CA_BUNDLE} + rules: + - operations: [ "UPDATE" ] + apiGroups: [ "apps", "" ] + apiVersions: ["v1beta1"] + resources: ["statefulsets"] diff --git a/pkg/label/label.go b/pkg/label/label.go index e453027d92d..17ea6c6d951 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -50,6 +50,8 @@ const ( AnnPVCDeferDeleting = "tidb.pingcap.com/pvc-defer-deleting" // AnnPVCPodScheduling is pod scheduling annotation key, it represents whether the pod is scheduling AnnPVCPodScheduling = "tidb.pingcap.com/pod-scheduling" + // AnnTiDBPartition is pod annotation which TiDB pod chould upgrade to + AnnTiDBPartition string = "tidb.pingcap.com/tidb-partition" // PDLabelVal is PD label value PDLabelVal string = "pd" diff --git a/pkg/tkctl/cmd/cmd.go b/pkg/tkctl/cmd/cmd.go index 9db7ac5f2de..ac3f858a2cc 100644 --- a/pkg/tkctl/cmd/cmd.go +++ b/pkg/tkctl/cmd/cmd.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/get" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/info" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/list" + "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/upinfo" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/use" "github.com/pingcap/tidb-operator/pkg/tkctl/config" "github.com/spf13/cobra" @@ -69,6 +70,7 @@ func NewTkcCommand(streams genericclioptions.IOStreams) *cobra.Command { info.NewCmdInfo(tkcContext, streams), use.NewCmdUse(tkcContext, streams), version.NewCmdVersion(tkcContext, streams.Out), + upinfo.NewCmdUpInfo(tkcContext, streams), }, }, { diff --git a/pkg/tkctl/cmd/upinfo/upinfo.go b/pkg/tkctl/cmd/upinfo/upinfo.go new file mode 100644 index 00000000000..b49ea965a9d --- /dev/null +++ b/pkg/tkctl/cmd/upinfo/upinfo.go @@ -0,0 +1,213 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package upinfo + +import ( + "fmt" + "io" + + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/tkctl/config" + "github.com/pingcap/tidb-operator/pkg/tkctl/readable" + "github.com/pingcap/tidb-operator/pkg/util" + "github.com/spf13/cobra" + apps "k8s.io/api/apps/v1beta1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +const ( + upinfoLongDesc = ` + Get tidb cluster component upgrade info. + + You can omit --tidbcluster= option by running 'tkc use ', +` + upinfoExample = ` + # get current tidb cluster info (set by tkc user) + tkc upinfo + + # get specified tidb cluster component upgrade info + tkc upinfo -t another-cluster +` + infoUsage = `expected 'upinfo -t CLUSTER_NAME' for the upinfo command or +using 'tkc use' to set tidb cluster first. +` + UPDATED = "updated" + UPDATING = "updating" + WAITING = "waiting" +) + +// UpInfoOptions contains the input to the list command. +type UpInfoOptions struct { + TidbClusterName string + Namespace string + + TcCli *versioned.Clientset + KubeCli *kubernetes.Clientset + + genericclioptions.IOStreams +} + +// NewUpInfoOptions returns a UpInfoOptions +func NewUpInfoOptions(streams genericclioptions.IOStreams) *UpInfoOptions { + return &UpInfoOptions{ + IOStreams: streams, + } +} + +// NewCmdUpInfo creates the upinfo command which show the tidb cluster upgrade detail information +func NewCmdUpInfo(tkcContext *config.TkcContext, streams genericclioptions.IOStreams) *cobra.Command { + o := NewUpInfoOptions(streams) + + cmd := &cobra.Command{ + Use: "upinfo", + Short: "Show tidb upgrade info.", + Example: upinfoExample, + Long: upinfoLongDesc, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(tkcContext, cmd, args)) + cmdutil.CheckErr(o.Run()) + }, + SuggestFor: []string{"updateinfo", "upgradeinfo"}, + } + + return cmd +} + +func (o *UpInfoOptions) Complete(tkcContext *config.TkcContext, cmd *cobra.Command, args []string) error { + + clientConfig, err := tkcContext.ToTkcClientConfig() + if err != nil { + return err + } + + if tidbClusterName, ok := clientConfig.TidbClusterName(); ok { + o.TidbClusterName = tidbClusterName + } else { + return cmdutil.UsageErrorf(cmd, infoUsage) + } + + namespace, _, err := clientConfig.Namespace() + if err != nil { + return err + } + o.Namespace = namespace + + restConfig, err := clientConfig.RestConfig() + if err != nil { + return err + } + tcCli, err := versioned.NewForConfig(restConfig) + if err != nil { + return err + } + o.TcCli = tcCli + kubeCli, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + o.KubeCli = kubeCli + + return nil +} + +func (o *UpInfoOptions) Run() error { + + tc, err := o.TcCli.PingcapV1alpha1(). + TidbClusters(o.Namespace). + Get(o.TidbClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + setName := controller.TiDBMemberName(tc.Name) + set, err := o.KubeCli.AppsV1beta1().StatefulSets(o.Namespace).Get(setName, metav1.GetOptions{}) + if err != nil { + return err + } + podList, err := o.KubeCli.CoreV1().Pods(o.Namespace).List(metav1.ListOptions{ + LabelSelector: label.New().Instance(tc.Name).TiDB().String(), + }) + if err != nil { + return err + } + msg, err := renderTCUpgradeInfo(tc, set, podList) + if err != nil { + return err + } + fmt.Fprint(o.Out, msg) + return nil +} + +func renderTCUpgradeInfo(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, podList *v1.PodList) (string, error) { + return readable.TabbedString(func(out io.Writer) error { + w := readable.NewPrefixWriter(out) + dbPhase := tc.Status.TiDB.Phase + w.WriteLine(readable.LEVEL_0, "Name:\t%s", tc.Name) + w.WriteLine(readable.LEVEL_0, "Namespace:\t%s", tc.Namespace) + w.WriteLine(readable.LEVEL_0, "CreationTimestamp:\t%s", tc.CreationTimestamp) + w.WriteLine(readable.LEVEL_0, "Status:\t%s", dbPhase) + if dbPhase == v1alpha1.UpgradePhase { + if len(podList.Items) != 0 { + pod := podList.Items[0] + w.WriteLine(readable.LEVEL_0, "Image:\t%s ---> %s", pod.Spec.Containers[0].Image, tc.Spec.TiDB.Image) + } + } + { + w.WriteLine(readable.LEVEL_1, "Name\tState\t") + w.WriteLine(readable.LEVEL_1, "----\t-----\t") + { + updateReplicas := set.Spec.UpdateStrategy.RollingUpdate.Partition + + if len(podList.Items) != 0 { + for _, pod := range podList.Items { + var state string + ordinal, err := util.GetOrdinalFromPodName(pod.Name) + if err != nil { + return err + } + if dbPhase == v1alpha1.UpgradePhase { + if (*updateReplicas) < ordinal { + state = UPDATED + } else if (*updateReplicas) == ordinal { + + state = UPDATING + + if pod.Labels[apps.ControllerRevisionHashLabelKey] == tc.Status.TiDB.StatefulSet.UpdateRevision { + if member, exist := tc.Status.TiDB.Members[pod.Name]; exist && member.Health { + state = UPDATED + } + } + + } else { + state = WAITING + } + } else { + state = UPDATED + } + w.WriteLine(readable.LEVEL_1, "%s\t%s\t", pod.Name, state) + } + } else { + w.WriteLine(readable.LEVEL_1, "no resource found") + } + } + } + return nil + }) +} diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go new file mode 100644 index 00000000000..b45cbece421 --- /dev/null +++ b/pkg/webhook/route/route.go @@ -0,0 +1,97 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package route + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/webhook/statefulset" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" +) + +// admitFunc is the type we use for all of our validators +type admitFunc func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse + +// marshal responseAdmissionReview and send back +func marshalAndWrite(response v1beta1.AdmissionReview, w http.ResponseWriter) { + + respBytes, err := json.Marshal(response) + if err != nil { + glog.Errorf("%v", err) + } + if _, err := w.Write(respBytes); err != nil { + glog.Errorf("%v", err) + } + +} + +// serve handles the http portion of a request prior to handing to an admit +// function +func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { + var body []byte + var contentType string + responseAdmissionReview := v1beta1.AdmissionReview{} + requestedAdmissionReview := v1beta1.AdmissionReview{} + deserializer := util.GetCodec() + + // The AdmissionReview that will be returned + if r.Body == nil { + err := errors.New("requeset body is nil") + responseAdmissionReview.Response = util.ARFail(err) + marshalAndWrite(responseAdmissionReview, w) + return + } + + data, err := ioutil.ReadAll(r.Body) + + if err != nil { + responseAdmissionReview.Response = util.ARFail(err) + marshalAndWrite(responseAdmissionReview, w) + return + } + + body = data + + // verify the content type is accurate + contentType = r.Header.Get("Content-Type") + if contentType != "application/json" { + err := errors.New("expect application/json") + responseAdmissionReview.Response = util.ARFail(err) + marshalAndWrite(responseAdmissionReview, w) + return + } + + // The AdmissionReview that was sent to the webhook + if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil { + responseAdmissionReview.Response = util.ARFail(err) + } else { + // pass to admitFunc + responseAdmissionReview.Response = admit(requestedAdmissionReview) + } + + // Return the same UID + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + + marshalAndWrite(responseAdmissionReview, w) + +} + +func ServeStatefulSets(w http.ResponseWriter, r *http.Request) { + serve(w, r, statefulset.AdmitStatefulSets) +} diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go new file mode 100644 index 00000000000..3612b513ff9 --- /dev/null +++ b/pkg/webhook/statefulset/statefulset.go @@ -0,0 +1,100 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "errors" + "fmt" + "strconv" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" + apps "k8s.io/api/apps/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" +) + +var ( + versionCli versioned.Interface + deserializer runtime.Decoder +) + +func init() { + deserializer = util.GetCodec() +} + +func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + + name := ar.Request.Name + namespace := ar.Request.Namespace + glog.Infof("admit statefulsets [%s/%s]", name, namespace) + + setResource := metav1.GroupVersionResource{Group: "apps", Version: "v1beta1", Resource: "statefulsets"} + if ar.Request.Resource != setResource { + err := fmt.Errorf("expect resource to be %s", setResource) + glog.Errorf("%v", err) + return util.ARFail(err) + } + + if versionCli == nil { + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Errorf("failed to get config: %v", err) + return util.ARFail(err) + } + + versionCli, err = versioned.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to create Clientset: %v", err) + return util.ARFail(err) + } + } + + raw := ar.Request.OldObject.Raw + set := apps.StatefulSet{} + if _, _, err := deserializer.Decode(raw, nil, &set); err != nil { + glog.Errorf("deseriralizer fail to decode request %v", err) + return util.ARFail(err) + } + + tc, err := versionCli.PingcapV1alpha1().TidbClusters(namespace).Get(set.Labels[label.InstanceLabelKey], metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to fetch tidbcluster info namespace %s clustername(instance) %s err %v", namespace, set.Labels[label.InstanceLabelKey], err) + return util.ARFail(err) + } + + if set.Labels[label.ComponentLabelKey] == label.TiDBLabelVal { + protect, ok := tc.Annotations[label.AnnTiDBPartition] + + if ok { + partition, err := strconv.ParseInt(protect, 10, 32) + if err != nil { + glog.Errorf("fail to convert protect to int namespace %s name %s err %v", namespace, name, err) + return util.ARFail(err) + } + + if (*set.Spec.UpdateStrategy.RollingUpdate.Partition) <= int32(partition) && tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { + glog.Infof("set has been protect by annotations name %s namespace %s", name, namespace) + return util.ARFail(errors.New("protect by annotation")) + } + } + } + + return util.ARSuccess() +} diff --git a/pkg/webhook/util/scheme.go b/pkg/webhook/util/scheme.go new file mode 100644 index 00000000000..d655a35940b --- /dev/null +++ b/pkg/webhook/util/scheme.go @@ -0,0 +1,39 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() + +func init() { + addToScheme(scheme) +} + +func addToScheme(scheme *runtime.Scheme) { + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) +} + +func GetCodec() runtime.Decoder { + return serializer.NewCodecFactory(scheme).UniversalDeserializer() +} diff --git a/pkg/webhook/util/util.go b/pkg/webhook/util/util.go new file mode 100644 index 00000000000..7f9bd143de6 --- /dev/null +++ b/pkg/webhook/util/util.go @@ -0,0 +1,51 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/tls" + + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ARFail is a helper function to create an AdmissionResponse +// with an embedded error +func ARFail(err error) *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: err.Error(), + Reason: metav1.StatusReasonNotAcceptable, + }, + } +} + +// ARSuccess return allow to action +func ARSuccess() *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: true, + } +} + +// config tls cert for server +func ConfigTLS(certFile string, keyFile string) (*tls.Config, error) { + sCert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + }, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 00000000000..24a1d53e904 --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,63 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "net/http" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/webhook/route" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/client-go/kubernetes" +) + +type WebhookServer struct { + // kubernetes client interface + KubeCli kubernetes.Interface + // operator client interface + Cli versioned.Interface + // http server + Server *http.Server +} + +func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, certFile string, keyFile string) *WebhookServer { + + http.HandleFunc("/statefulsets", route.ServeStatefulSets) + + sCert, err := util.ConfigTLS(certFile, keyFile) + + if err != nil { + glog.Fatalf("failed to create scert file %v", err) + } + + server := &http.Server{ + Addr: ":443", + TLSConfig: sCert, + } + + return &WebhookServer{ + KubeCli: kubecli, + Cli: cli, + Server: server, + } +} + +func (ws *WebhookServer) Run() error { + return ws.Server.ListenAndServeTLS("", "") +} + +func (ws *WebhookServer) Shutdown() error { + return ws.Server.Shutdown(nil) +} diff --git a/tests/actions.go b/tests/actions.go index d9b5184947f..d92f61e4540 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -127,7 +127,6 @@ type OperatorActions interface { CheckScaledCorrectly(info *TidbClusterConfig, podUIDsBeforeScale map[string]types.UID) error UpgradeTidbCluster(info *TidbClusterConfig) error UpgradeTidbClusterOrDie(info *TidbClusterConfig) - CheckUpgradeProgress(info *TidbClusterConfig) error DeployAdHocBackup(info *TidbClusterConfig) error CheckAdHocBackup(info *TidbClusterConfig) error DeployScheduledBackup(info *TidbClusterConfig) error @@ -367,6 +366,33 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { return fmt.Errorf("failed to deploy operator: %v, %s", err, string(res)) } + // create cert and secret for webhook + cmd = fmt.Sprintf("%s/create-cert.sh", oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create cert: %v, %s", err, string(res)) + } + + // patch cabundle to validating admission configuration + cmd = fmt.Sprintf("%s/patch-ca.sh", oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to patch cabundle : %v, %s", err, string(res)) + } + + // deploy statefulset webhook and configuration to hijack update statefulset opeartion + cmd = fmt.Sprintf("kubectl apply -f %s/webhook.yaml", oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create statefulset webhook and configuration : %v, %s", err, string(res)) + } + return nil } @@ -384,7 +410,16 @@ func (oa *operatorActions) CleanOperator(info *OperatorConfig) error { return err } - res, err := exec.Command("helm", "del", "--purge", info.ReleaseName).CombinedOutput() + // delete statefulset update webhook and configuration + cmd := fmt.Sprintf("kubectl delete -f %s/webhook.yaml", oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil && !notFound(string(res)) { + return fmt.Errorf("failed to delete statefulset webhook and configuration : %v, %s", err, string(res)) + } + + res, err = exec.Command("helm", "del", "--purge", info.ReleaseName).CombinedOutput() if err == nil || !releaseIsNotFound(err) { return nil @@ -575,7 +610,7 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error ns := info.Namespace tcName := info.ClusterName - if err := wait.Poll(oa.pollInterval, 30*time.Minute, func() (bool, error) { + if err := wait.Poll(oa.pollInterval, 35*time.Minute, func() (bool, error) { var tc *v1alpha1.TidbCluster var err error if tc, err = oa.cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{}); err != nil { @@ -675,6 +710,10 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) { info.blockWriter.Stop() } +func (oa *operatorActions) manifestPath(tag string) string { + return filepath.Join(oa.cfg.ManifestDir, tag) +} + func (oa *operatorActions) chartPath(name string, tag string) string { return filepath.Join(oa.cfg.ChartDir, tag, name) } @@ -768,6 +807,18 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs }) } +func setPartitionAnnotation(tcName string, nameSpace string, ordinal int) error { + // add annotation to pause statefulset upgrade process + cmd := fmt.Sprintf("kubectl annotate tc %s -n %s tidb.pingcap.com/tidb-partition=%d --overwrite", + tcName, nameSpace, ordinal) + glog.Infof("%s", cmd) + _, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return err + } + return nil +} + func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { // record tikv leader count in webhook first err := webhook.GetAllKVLeaders(oa.cli, info.Namespace, info.ClusterName) @@ -776,6 +827,19 @@ func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { } oa.EmitEvent(info, "UpgradeTidbCluster") + // get tidbSet from apiserver + tidbSetName := controller.TiDBMemberName(info.ClusterName) + tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tidbSetName, metav1.GetOptions{}) + if err != nil { + return pingcapErrors.Wrapf(err, "failed to get stateful set [%s/%s] setName %s", info.Namespace, info.ClusterName, tidbSetName) + } + + // add annotation to pause statefulset upgrade process + err = setPartitionAnnotation(info.ClusterName, info.Namespace, int(tidbSet.Status.Replicas-1)) + if err != nil { + return pingcapErrors.Wrapf(err, "failed to add annotation to [%s/%s]", info.Namespace, info.ClusterName) + } + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[UPGRADE] " + cmd) @@ -792,157 +856,29 @@ func (oa *operatorActions) UpgradeTidbClusterOrDie(info *TidbClusterConfig) { } } -func (oa *operatorActions) CheckUpgradeProgress(info *TidbClusterConfig) error { - return wait.Poll(oa.pollInterval, DefaultPollTimeout, func() (done bool, err error) { - tc, err := oa.cli.PingcapV1alpha1().TidbClusters(info.Namespace).Get(info.ClusterName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tidbcluster: [%s], error: %v", info.ClusterName, err) - return false, nil - } - - pdSetName := controller.PDMemberName(info.ClusterName) - pdSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(pdSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get pd statefulset: [%s], error: %v", pdSetName, err) - return false, nil - } - - tikvSetName := controller.TiKVMemberName(info.ClusterName) - tikvSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tikvSet statefulset: [%s], error: %v", tikvSetName, err) - return false, nil - } - - tidbSetName := controller.TiDBMemberName(info.ClusterName) - tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tidbSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tidbSet statefulset: [%s], error: %v", tidbSetName, err) - return false, nil - } - - imageUpgraded := func(memberType v1alpha1.MemberType, set *v1beta1.StatefulSet) bool { - image := "" - switch memberType { - case v1alpha1.PDMemberType: - image = tc.Spec.PD.Image - case v1alpha1.TiKVMemberType: - image = tc.Spec.TiKV.Image - case v1alpha1.TiDBMemberType: - image = tc.Spec.TiDB.Image - } - memberName := string(memberType) - c, ok := getComponentContainer(set) - if !ok || c.Image != image { - glog.Infof("check %s image: getContainer(set).Image(%s) != tc.Spec.%s.Image(%s)", - memberName, c.Image, strings.ToUpper(memberName), image) - } - return ok && c.Image == image - } - setUpgraded := func(set *v1beta1.StatefulSet) bool { - return set.Generation <= *set.Status.ObservedGeneration && set.Status.CurrentRevision == set.Status.UpdateRevision - } - - // check upgrade order - if tc.Status.PD.Phase == v1alpha1.UpgradePhase { - glog.Infof("pd is upgrading") - if tc.Status.TiKV.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tikv is upgrading while pd is upgrading") - } - if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tidb is upgrading while pd is upgrading") - } - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while pd is upgrading") - } - if !setUpgraded(pdSet) { - if imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is updated while pd is upgrading") - } - if imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is updated while pd is upgrading") - } - } - return false, nil - } else if tc.Status.TiKV.Phase == v1alpha1.UpgradePhase { - glog.Infof("tikv is upgrading") - if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tidb is upgrading while tikv is upgrading") - } - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while tikv is upgrading") - } - if !setUpgraded(pdSet) { - return false, pingcapErrors.New("pd stateful set is not upgraded while tikv is upgrading") - } - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is not updated while tikv is upgrading") - } - if !setUpgraded(tikvSet) { - if imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is updated while tikv is upgrading") - } - } - return false, nil - } else if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - glog.Infof("tidb is upgrading") - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while tidb is upgrading") - } - if !setUpgraded(pdSet) { - return false, pingcapErrors.New("pd stateful set is not upgraded while tidb is upgrading") - } - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is not updated while tidb is upgrading") - } - if !setUpgraded(tikvSet) { - return false, pingcapErrors.New("tikv stateful set is not upgraded while tidb is upgrading") - } - if !imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is not updated while tikv is upgrading") - } - return false, nil - } - - // check pd final state - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, nil - } - if !setUpgraded(pdSet) { - glog.Infof("check pd stateful set upgraded failed") - return false, nil - } - // check tikv final state - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, nil - } - if !setUpgraded(tikvSet) { - glog.Infof("check tikv stateful set upgraded failed") - return false, nil - } - // check tidb final state - if !imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, nil - } - if !setUpgraded(tidbSet) { - glog.Infof("check tidb stateful set upgraded failed") - return false, nil - } - return true, nil - }) -} - func (oa *operatorActions) DeployMonitor(info *TidbClusterConfig) error { return nil } func (oa *operatorActions) CleanMonitor(info *TidbClusterConfig) error { return nil } -func getComponentContainer(set *v1beta1.StatefulSet) (corev1.Container, bool) { - name := set.Labels[label.ComponentLabelKey] - for _, c := range set.Spec.Template.Spec.Containers { - if c.Name == name { - return c, true +func getMemberContainer(kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container, bool) { + name := fmt.Sprintf("%s-%d", memberName, 0) + pod, err := kubeCli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to get pod [%s/%s]", namespace, name) + return nil, false + } + if len(pod.Spec.Containers) == 0 { + glog.Errorf("no container in this pod [%s/%s]", namespace, name) + return nil, false + } + + for _, container := range pod.Spec.Containers { + if container.Name == v1alpha1.PDMemberType.String() || + container.Name == v1alpha1.TiKVMemberType.String() || + container.Name == v1alpha1.TiDBMemberType.String() { + return &container, true } } - return corev1.Container{}, false + return nil, false } func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) { @@ -982,7 +918,7 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err ns, pdSetName, pdSet.Status.ReadyReplicas, pdSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(pdSet); !ok || tc.Spec.PD.Image != c.Image { + if c, ok := getMemberContainer(oa.kubeCli, ns, pdSetName); !ok || tc.Spec.PD.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=pd].image(%s) != %s", ns, pdSetName, c.Image, tc.Spec.PD.Image) return false, nil @@ -1047,7 +983,7 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tikvSetName, tikvSet.Status.ReadyReplicas, tikvSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(tikvSet); !ok || tc.Spec.TiKV.Image != c.Image { + if c, ok := getMemberContainer(oa.kubeCli, ns, tikvSetName); !ok || tc.Spec.TiKV.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", ns, tikvSetName, c.Image, tc.Spec.TiKV.Image) return false, nil @@ -1073,6 +1009,38 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e tcName := tc.GetName() ns := tc.GetNamespace() tidbSetName := controller.TiDBMemberName(tcName) + tidbUpgradeAnnotationStr, ok := tc.Annotations[label.AnnTiDBPartition] + if !ok { + tidbUpgradeAnnotationStr = "0" + } + + tidbUpgradeAnnotation, err := strconv.ParseInt(tidbUpgradeAnnotationStr, 10, 32) + if err != nil { + return false, nil + } + + pauseCorrect := func(set *v1beta1.StatefulSet) bool { + return (*set.Spec.UpdateStrategy.RollingUpdate.Partition) >= int32(tidbUpgradeAnnotation) + } + + upgradePaused := func() bool { + + podName := fmt.Sprintf("%s-%d", controller.TiDBMemberName(tc.Name), tidbUpgradeAnnotation) + + tidbPod, err := oa.kubeCli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to get tidb po name %s namespace %s ", podName, ns) + return false + } + if tidbPod.Labels[v1beta1.ControllerRevisionHashLabelKey] == tc.Status.TiDB.StatefulSet.UpdateRevision && + tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { + if member, ok := tc.Status.TiDB.Members[tidbPod.Name]; ok && member.Health { + return true + } + } + + return false + } tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{}) if err != nil { @@ -1106,8 +1074,26 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tidbSetName, tidbSet.Status.ReadyReplicas, tidbSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(tidbSet); !ok || tc.Spec.TiDB.Image != c.Image { - glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", + + if upgradePaused() { + + time.Sleep(5 * time.Minute) + + if !pauseCorrect(tidbSet) { + return false, fmt.Errorf("pause partition is not correct in upgrade phase [%s/%s] partition %d annotation %d", + ns, tidbSetName, (*tidbSet.Spec.UpdateStrategy.RollingUpdate.Partition), tidbUpgradeAnnotation) + } + + err := setPartitionAnnotation(tcName, ns, 0) + if err != nil { + glog.Errorf("fail to set annotation for [%s/%s]", ns, tidbSetName) + return false, nil + } + return false, nil + } + + if c, ok := getMemberContainer(oa.kubeCli, ns, tidbSetName); !ok || tc.Spec.TiDB.Image != c.Image { + glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tidb].image(%s) != %s", ns, tidbSetName, c.Image, tc.Spec.TiDB.Image) return false, nil } @@ -1592,6 +1578,10 @@ func releaseIsNotFound(err error) bool { return strings.Contains(err.Error(), "not found") } +func notFound(res string) bool { + return strings.Contains(res, "not found") +} + func (oa *operatorActions) cloneOperatorRepo() error { cmd := fmt.Sprintf("git clone https://github.com/pingcap/tidb-operator.git %s", oa.cfg.OperatorRepoDir) glog.Info(cmd) @@ -1606,10 +1596,12 @@ func (oa *operatorActions) cloneOperatorRepo() error { func (oa *operatorActions) checkoutTag(tagName string) error { cmd := fmt.Sprintf("cd %s && git stash -u && git checkout %s && "+ "mkdir -p %s && cp -rf charts/tidb-operator %s && "+ - "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s", + "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s &&"+ + "cp -rf manifests %s", oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), - oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName)) + oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName), + oa.manifestPath(tagName)) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index a20a021f550..7aa143fb8fc 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -32,6 +32,7 @@ func main() { conf := tests.ParseConfigOrDie() conf.ChartDir = "/charts" + conf.ManifestDir = "/manifests" cli, kubeCli := client.NewCliOrDie() oa := tests.NewOperatorActions(cli, kubeCli, 5*time.Second, conf, nil) diff --git a/tests/config.go b/tests/config.go index 1fed05a369e..c907a224cf0 100644 --- a/tests/config.go +++ b/tests/config.go @@ -48,6 +48,8 @@ type Config struct { OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"` // chart dir ChartDir string `yaml:"chart_dir" json:"chart_dir"` + // manifest dir + ManifestDir string `yaml:"manifest_dir" json:"manifest_dir"` } // Nodes defines a series of nodes that belong to the same physical node. @@ -92,6 +94,13 @@ func NewConfig() (*Config, error) { return nil, err } cfg.ChartDir = chartDir + + manifestDir, err := ioutil.TempDir("", "manifests") + if err != nil { + return nil, err + } + cfg.ManifestDir = manifestDir + return cfg, nil } diff --git a/tests/images/e2e/Dockerfile b/tests/images/e2e/Dockerfile index 5abe477cd39..7294d5bc953 100644 --- a/tests/images/e2e/Dockerfile +++ b/tests/images/e2e/Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.5 ENV KUBECTL_VERSION=v1.12.2 ENV HELM_VERSION=v2.9.1 -RUN apk update && apk add --no-cache ca-certificates curl git +RUN apk update && apk add --no-cache ca-certificates curl git openssl bash RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ -o /usr/local/bin/kubectl && \ chmod +x /usr/local/bin/kubectl && \ @@ -17,5 +17,6 @@ RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER ADD tidb-operator /charts/e2e/tidb-operator ADD tidb-cluster /charts/e2e/tidb-cluster ADD tidb-backup /charts/e2e/tidb-backup +ADD manifests /manifests/e2e ADD bin/e2e /usr/local/bin/e2e