From 9f52eddbfe900499b6792420d5760fdf1669983c Mon Sep 17 00:00:00 2001 From: onlymellb Date: Tue, 10 Dec 2019 17:33:43 +0800 Subject: [PATCH] Automated cherry pick of #1237: put release jenkinsfile from jenkins into repo (#1318) --- ci/pingcap_tidb_operator_build_kind.groovy | 49 +++--- ci/release_cli_binary_and_debug_image.groovy | 162 ++++++++++++------ ...idb_operator_binary_and_docker_dind.groovy | 74 -------- ...ease_tidb_operator_binary_and_image.groovy | 112 ++++++++++++ 4 files changed, 244 insertions(+), 153 deletions(-) delete mode 100644 ci/release_tidb_operator_binary_and_docker_dind.groovy create mode 100644 ci/release_tidb_operator_binary_and_image.groovy diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy index 1fc6b6efb3c..1b8ef019cca 100644 --- a/ci/pingcap_tidb_operator_build_kind.groovy +++ b/ci/pingcap_tidb_operator_build_kind.groovy @@ -17,6 +17,7 @@ def getChangeLogText() { def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) { def GITHASH + def CODECOV_TOKEN def UCLOUD_OSS_URL = "http://pingcap-dev.hk.ufileos.com" def BUILD_URL = "git@github.com:pingcap/tidb-operator.git" def PROJECT_DIR = "go/src/github.com/pingcap/tidb-operator" @@ -44,25 +45,29 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) { GITHASH = sh(returnStdout: true, script: "git rev-parse HEAD").trim() withCredentials([string(credentialsId: "${CODECOV_CREDENTIALS_ID}", variable: 'codecovToken')]) { - sh """ - export GOPATH=${WORKSPACE}/go - export PATH=${WORKSPACE}/go/bin:\$PATH - if ! hash hg 2>/dev/null; then - sudo yum install -y mercurial - fi - hg --version - make check-setup - make check - if [ ${BUILD_BRANCH} == "master" ] - then - make test GO_COVER=y - curl -s https://codecov.io/bash | bash -s - -t ${codecovToken} || echo'Codecov did not collect coverage reports' - else - make test - fi - make - make e2e-build - """ + CODECOV_TOKEN = codecovToken + } + + ansiColor('xterm') { + sh """ + export GOPATH=${WORKSPACE}/go + export PATH=${WORKSPACE}/go/bin:\$PATH + if ! hash hg 2>/dev/null; then + sudo yum install -y mercurial + fi + hg --version + make check-setup + make check + if [ ${BUILD_BRANCH} == "master" ] + then + make test GO_COVER=y + curl -s https://codecov.io/bash | bash -s - -t ${CODECOV_TOKEN} || echo'Codecov did not collect coverage reports' + else + make test + fi + make + make e2e-build + """ } } } @@ -164,9 +169,9 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) { stage('Summary') { def CHANGELOG = getChangeLogText() def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def slackmsg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + + def slackmsg = "[#${env.ghprbPullId}: ${env.ghprbPullTitle}]" + "\n" + + "${env.ghprbPullLink}" + "\n" + + "${env.ghprbPullDescription}" + "\n" + "Integration Common Test Result: `${currentBuild.result}`" + "\n" + "Elapsed Time: `${duration} mins` " + "\n" + "${CHANGELOG}" + "\n" + diff --git a/ci/release_cli_binary_and_debug_image.groovy b/ci/release_cli_binary_and_debug_image.groovy index 06443a9a677..c28116aa9ca 100644 --- a/ci/release_cli_binary_and_debug_image.groovy +++ b/ci/release_cli_binary_and_debug_image.groovy @@ -1,60 +1,108 @@ -def call(BUILD_BRANCH, RELEASE_TAG) { - - env.GOPATH = "/go" - env.GOROOT = "/usr/local/go" - env.PATH = "${env.GOROOT}/bin:${env.GOPATH}/bin:/bin:${env.PATH}:/home/jenkins/bin" - - def GITHASH - def TKCTL_CLI_PACKAGE = "tkctl-${GOOS}-${GOARCH}-${RELEASE_TAG}" - - catchError { - node('k8s_centos7_build') { - def WORKSPACE = pwd() - - dir("${WORKSPACE}/operator"){ - stage('Build and release CLI to qiniu'){ - checkout([$class: 'GitSCM', branches: [[name: "${BUILD_BRANCH}"]], userRemoteConfigs:[[url: "${BUILD_URL}", credentialsId: "${CREDENTIALS_ID}"]]]) - GITHASH = sh(returnStdout: true, script: "git rev-parse HEAD").trim() - def GOARCH = "amd64" - ["linux", "darwin", "windows"].each { - sh """ - GOOS=${it} GOARCH=${GOARCH} make cli - tar -zcf ${TKCTL_CLI_PACKAGE}.tgz tkctl - sha256sum ${TKCTL_CLI_PACKAGE}.tgz > ${TKCTL_CLI_PACKAGE}.sha256 - - upload.py ${TKCTL_CLI_PACKAGE}.tgz ${TKCTL_CLI_PACKAGE}.tgz - upload.py ${TKCTL_CLI_PACKAGE}.sha256 ${TKCTL_CLI_PACKAGE}.sha256 - """ - } - } - - stage('Build and push debug images'){ - withDockerServer([uri: "${env.DOCKER_HOST}"]) { - DOCKER_REGISTRY="" make debug-docker-push - DOCKER_REGISTRY="uhub.service.ucloud.cn" make debug-docker-push - } - } - } - } - currentBuild.result = "SUCCESS" - } - stage('Summary') { - echo("echo summary info ########") - def DURATION = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def slackmsg = "[${env.JOB_NAME.replaceAll('%2F','/')}-${env.BUILD_NUMBER}] `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${DURATION}` Mins" + "\n" + - "tidb-operator Branch: `${BUILD_BRANCH}`, Githash: `${GITHASH.take(7)}`" + "\n" + - "Display URL:" + "\n" + - "${env.RUN_DISPLAY_URL}" - - if(currentBuild.result != "SUCCESS"){ - slackSend channel: '#cloud_jenkins', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" - } else { - slackmsg = "${slackmsg}" + "\n" + - "tkctl cli tool build and debug image build failed for BRANCH:${BUILD_BRANCH} and TAG:${RELEASE_TAG}`" - slackSend channel: '#cloud_jenkins', color: 'good', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" - } - } +def call(BUILD_BRANCH, RELEASE_TAG, CREDENTIALS_ID) { + def GITHASH + def TKCTL_CLI_PACKAGE + def GOARCH = "amd64" + def OS_LIST = ["linux", "darwin", "windows"] + def DEBUG_LIST = ["debug-launcher", "tidb-control", "tidb-debug"] + def BUILD_URL = "git@github.com:pingcap/tidb-operator.git" + def PROJECT_DIR = "go/src/github.com/pingcap/tidb-operator" + + catchError { + node('build_go1130_memvolume') { + container("golang") { + def WORKSPACE = pwd() + dir("${PROJECT_DIR}") { + stage('build tkcli') { + checkout([$class: 'GitSCM', branches: [[name: "${BUILD_BRANCH}"]], userRemoteConfigs: [[url: "${BUILD_URL}", credentialsId: "${CREDENTIALS_ID}"]]]) + checkout changelog: false, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "${BUILD_BRANCH}"]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: "${CREDENTIALS_ID}", + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + url: "${BUILD_URL}", + ]] + ] + + GITHASH = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + sh "make debug-build" + + OS_LIST.each { + TKCTL_CLI_PACKAGE = "tkctl-${it}-${GOARCH}-${RELEASE_TAG}" + sh """ + GOOS=${it} GOARCH=${GOARCH} make cli + tar -zcf ${TKCTL_CLI_PACKAGE}.tgz tkctl + sha256sum ${TKCTL_CLI_PACKAGE}.tgz > ${TKCTL_CLI_PACKAGE}.sha256 + """ + } + } + } + + stash excludes: "${PROJECT_DIR}/vendor/**", includes: "${PROJECT_DIR}/**", name: "tidb-operator" + } + } + node('delivery') { + container("delivery") { + def WORKSPACE = pwd() + sh "chown -R jenkins:jenkins ./" + deleteDir() + unstash 'tidb-operator' + + dir("${PROJECT_DIR}") { + stage('Publish tkcli') { + OS_LIST.each { + TKCTL_CLI_PACKAGE = "tkctl-${it}-${GOARCH}-${RELEASE_TAG}" + sh """ + upload.py ${TKCTL_CLI_PACKAGE}.tgz ${TKCTL_CLI_PACKAGE}.tgz + upload.py ${TKCTL_CLI_PACKAGE}.sha256 ${TKCTL_CLI_PACKAGE}.sha256 + """ + } + + stage('Push utility docker images') { + withDockerServer([uri: "${env.DOCKER_HOST}"]) { + DEBUG_LIST.each { + docker.build("uhub.service.ucloud.cn/pingcap/${it}:${RELEASE_TAG}", "misc/images/${it}").push() + docker.build("pingcap/${it}:${RELEASE_TAG}", "misc/images/${it}").push() + } + } + } + } + } + } + } + currentBuild.result = "SUCCESS" + } + stage('Summary') { + echo("echo summary info ########") + def DURATION = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) + def slackmsg = "[${env.JOB_NAME.replaceAll('%2F','/')}-${env.BUILD_NUMBER}] `${currentBuild.result}`" + "\n" + + "Elapsed Time: `${DURATION}` Mins" + "\n" + + "tidb-operator Branch: `${BUILD_BRANCH}`, Githash: `${GITHASH.take(7)}`" + "\n" + + "Display URL:" + "\n" + + "${env.RUN_DISPLAY_URL}" + + if(currentBuild.result != "SUCCESS"){ + slackSend channel: '#cloud_jenkins', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" + return + } + + DEBUG_LIST.each{ + slackmsg = "${slackmsg}" + "\n" + + "${it} Docker Image: Docker Image: `pingcap/${it}:${RELEASE_TAG}`" + "\n" + + "${it} Docker Image: Docker Image: `uhub.ucloud.cn/pingcap/${it}:${RELEASE_TAG}`" + } + + OS_LIST.each{ + slackmsg = "${slackmsg}" + "\n" + + "tkctl ${it} binary Download URL: https://download.pingcap.org/tkctl-${it}-${GOARCH}-${RELEASE_TAG}.tgz" + } + slackSend channel: '#cloud_jenkins', color: 'good', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" + } } return this diff --git a/ci/release_tidb_operator_binary_and_docker_dind.groovy b/ci/release_tidb_operator_binary_and_docker_dind.groovy deleted file mode 100644 index 283b05b93e1..00000000000 --- a/ci/release_tidb_operator_binary_and_docker_dind.groovy +++ /dev/null @@ -1,74 +0,0 @@ -def call(BUILD_BRANCH, RELEASE_TAG) { - - env.GOPATH = "/go" - env.GOROOT = "/usr/local/go" - env.PATH = "${env.GOROOT}/bin:${env.GOPATH}/bin:/bin:${env.PATH}:/home/jenkins/bin" - - def GITHASH - def UCLOUD_OSS_URL = "http://pingcap-dev.hk.ufileos.com" - def TIDB_OPERATOR_CHART = "tidb-operator-${RELEASE_TAG}" - def TIDB_CLUSTER_CHART = "tidb-cluster-${RELEASE_TAG}" - - catchError { - node('k8s_centos7_build') { - def WORKSPACE = pwd() - - dir("${WORKSPACE}/operator"){ - stage('Download tidb-operator binary'){ - GITHASH = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/operator/${BUILD_BRANCH}/centos7/sha1").trim() - sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/operator/${GITHASH}/centos7/tidb-operator.tar.gz | tar xz" - } - - stage('Push tidb-operator Docker Image'){ - withDockerServer([uri: "${env.DOCKER_HOST}"]) { - docker.build("uhub.service.ucloud.cn/pingcap/tidb-operator:${RELEASE_TAG}", "images/tidb-operator").push() - docker.build("pingcap/tidb-operator:${RELEASE_TAG}", "images/tidb-operator").push() - } - } - - stage('Release charts to qiniu'){ - sh """ - # release tidb-operator chart - sed -i "s/version:.*/version: ${RELEASE_TAG}/g" charts/tidb-operator/Chart.yaml - tar -zcf ${TIDB_OPERATOR_CHART}.tgz -C charts tidb-operator - sha256sum ${TIDB_OPERATOR_CHART}.tgz > ${TIDB_OPERATOR_CHART}.sha256 - - upload.py ${TIDB_OPERATOR_CHART}.tgz ${TIDB_OPERATOR_CHART}.tgz - upload.py ${TIDB_OPERATOR_CHART}.sha256 ${TIDB_OPERATOR_CHART}.sha256 - # release tidb-cluster chart - tar -zcf ${TIDB_CLUSTER_CHART}.tgz -C charts tidb-cluster - sha256sum ${TIDB_CLUSTER_CHART}.tgz > ${TIDB_CLUSTER_CHART}.sha256 - - upload.py ${TIDB_CLUSTER_CHART}.tgz ${TIDB_CLUSTER_CHART}.tgz - upload.py ${TIDB_CLUSTER_CHART}.sha256 ${TIDB_CLUSTER_CHART}.sha256 - """ - } - } - } - currentBuild.result = "SUCCESS" - } - stage('Summary') { - echo("echo summary info ########") - def DURATION = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def slackmsg = "[${env.JOB_NAME.replaceAll('%2F','/')}-${env.BUILD_NUMBER}] `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${DURATION}` Mins" + "\n" + - "tidb-operator Branch: `${BUILD_BRANCH}`, Githash: `${GITHASH.take(7)}`" + "\n" + - "Display URL:" + "\n" + - "${env.RUN_DISPLAY_URL}" - - if(currentBuild.result != "SUCCESS"){ - slackSend channel: '#cloud_jenkins', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" - } else { - slackmsg = "${slackmsg}" + "\n" + - "tidb-operator Docker Image: `pingcap/tidb-operator:${RELEASE_TAG}`" + "\n" + - "tidb-operator Docker Image: `uhub.ucloud.cn/pingcap/tidb-operator:${RELEASE_TAG}`" + "\n" + - "tidb-operator charts Download URL: http://download.pingcap.org/${TIDB_OPERATOR_CHART}.tgz" + "\n" + - "tidb-operator charts sha256: http://download.pingcap.org/${TIDB_OPERATOR_CHART}.sha256" + "\n" + - "tidb-cluster charts Download URL: http://download.pingcap.org/${TIDB_CLUSTER_CHART}.tgz" + "\n" + - "tidb-cluster charts sha256: http://download.pingcap.org/${TIDB_CLUSTER_CHART}.sha256" - slackSend channel: '#cloud_jenkins', color: 'good', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" - } - } -} - -return this diff --git a/ci/release_tidb_operator_binary_and_image.groovy b/ci/release_tidb_operator_binary_and_image.groovy new file mode 100644 index 00000000000..8b8b117b72a --- /dev/null +++ b/ci/release_tidb_operator_binary_and_image.groovy @@ -0,0 +1,112 @@ +def call(BUILD_BRANCH, RELEASE_TAG, CREDENTIALS_ID, CHART_ITEMS) { + def GITHASH + def ACCESS_KEY + def SECRET_KEY + def UCLOUD_OSS_URL = "http://pingcap-dev.hk.ufileos.com" + + catchError { + node('delivery') { + container("delivery") { + def WORKSPACE = pwd() + withCredentials([string(credentialsId: "${env.QN_ACCESS_KET_ID}", variable: 'QN_access_key'), string(credentialsId: "${env.QN_SECRET_KEY_ID}", variable: 'Qiniu_secret_key')]) { + ACCESS_KEY = QN_access_key + SECRET_KEY = Qiniu_secret_key + } + + sh "chown -R jenkins:jenkins ./" + deleteDir() + + dir("${WORKSPACE}/operator"){ + stage('Download tidb-operator binary'){ + GITHASH = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/operator/${BUILD_BRANCH}/centos7/sha1").trim() + sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/operator/${GITHASH}/centos7/tidb-operator.tar.gz | tar xz" + } + stage('Push tidb-operator Docker Image'){ + withDockerServer([uri: "${env.DOCKER_HOST}"]) { + docker.build("uhub.service.ucloud.cn/pingcap/tidb-operator:${RELEASE_TAG}", "images/tidb-operator").push() + docker.build("pingcap/tidb-operator:${RELEASE_TAG}", "images/tidb-operator").push() + } + } + + stage('Push tidb-backup-manager Docker Image'){ + withDockerServer([uri: "${env.DOCKER_HOST}"]) { + docker.build("uhub.service.ucloud.cn/pingcap/backup-manager:${RELEASE_TAG}", "images/backup-manager").push() + docker.build("pingcap/backup-manager:${RELEASE_TAG}", "images/backup-manager").push() + } + } + + stage('Release charts to qiniu'){ + ansiColor('xterm') { + sh """ + set +x + export QINIU_ACCESS_KEY="${ACCESS_KEY}" + export QINIU_SECRET_KEY="${SECRET_KEY}" + export QINIU_BUCKET_NAME="charts" + set -x + curl https://raw.githubusercontent.com/pingcap/docs-cn/master/scripts/upload.py -o upload.py + sed -i 's%http://download.pingcap.org%http://charts.pingcap.org%g' upload.py + sed -i 's/python3/python/g' upload.py + chmod +x upload.py + for chartItem in ${CHART_ITEMS} + do + chartPrefixName=\$chartItem-${RELEASE_TAG} + echo "======= release \$chartItem chart ======" + sed -i "s/version:.*/version: ${RELEASE_TAG}/g" charts/\$chartItem/Chart.yaml + tar -zcf \${chartPrefixName}.tgz -C charts \$chartItem + sha256sum \${chartPrefixName}.tgz > \${chartPrefixName}.sha256 + ./upload.py \${chartPrefixName}.tgz \${chartPrefixName}.tgz + ./upload.py \${chartPrefixName}.sha256 \${chartPrefixName}.sha256 + done + #Generate index.yaml for helm repo + wget https://storage.googleapis.com/kubernetes-helm/helm-v2.14.1-linux-amd64.tar.gz + tar -zxvf helm-v2.14.1-linux-amd64.tar.gz + mv linux-amd64/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + #ls + curl http://charts.pingcap.org/index.yaml -o index.yaml + cat index.yaml + helm repo index . --url http://charts.pingcap.org/ --merge index.yaml + cat index.yaml + ./upload.py index.yaml index.yaml + """ + } + } + } + } + } + currentBuild.result = "SUCCESS" + } + + stage('Summary') { + echo("echo summary info ########") + def DURATION = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) + def slackmsg = "[${env.JOB_NAME.replaceAll('%2F','/')}-${env.BUILD_NUMBER}] `${currentBuild.result}`" + "\n" + + "Elapsed Time: `${DURATION}` Mins" + "\n" + + "tidb-operator Branch: `${BUILD_BRANCH}`, Githash: `${GITHASH.take(7)}`" + "\n" + + "Display URL:" + "\n" + + "${env.RUN_DISPLAY_URL}" + + if(currentBuild.result != "SUCCESS"){ + slackSend channel: '#cloud_jenkins', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" + return + } + + slackmsg = "${slackmsg}" + "\n" + + "tidb-operator Docker Image: `pingcap/tidb-operator:${RELEASE_TAG}`" + "\n" + + "tidb-operator Docker Image: `uhub.ucloud.cn/pingcap/tidb-operator:${RELEASE_TAG}`" + "\n" + + "backup-manager Docker Image: `pingcap/backup-manager:${RELEASE_TAG}`" + "\n" + + "backup-manager Docker Image: `uhub.ucloud.cn/pingcap/backup-manager:${RELEASE_TAG}`" + + + for(String chartItem : CHART_ITEMS.split(' ')){ + slackmsg = "${slackmsg}" + "\n" + + "${chartItem} charts Download URL: http://charts.pingcap.org/${chartItem}-${RELEASE_TAG}.tgz" + } + slackmsg = "${slackmsg}" + "\n" + + "charts index Download URL: http://charts.pingcap.org/index.yaml" + + slackSend channel: '#cloud_jenkins', color: 'good', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" + } +} + +return this