diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml new file mode 100644 index 00000000000..70019774657 --- /dev/null +++ b/.github/workflows/blossom-ci.yml @@ -0,0 +1,114 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A workflow to trigger blossom-CI on self-hosted runner +name: Blossom-CI +on: + issue_comment: + types: [created] + +jobs: + authorization: + name: Authorization + # trigger on pre-defined text + if: github.event.comment.body == 'build' + runs-on: [self-hosted, linux, blossom] + steps: + - name: Check if comment is issued by authorized person + run: blossom-ci + env: + OPERATION: 'AUTH' + VERSION: '1' + + vulnerability-scan-job: + name: Vulnerability scan job + needs: [authorization] + runs-on: ubuntu-latest + steps: + - name: Get pull request data + id: pull_request_data + uses: octokit/request-action@v2.x + with: + route: 'GET /repos/:repository/pulls/:issue_id' + repository: ${{ github.repository }} + issue_id: ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Set blackduck project version + id: blackduck-project-version + run: echo "${{ fromJson(steps.pull_request_data.outputs.data).head.ref }}-${{ github.run_id }}" + + - name: Update status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/:repository/statuses/:sha' + repository: ${{ github.repository }} + sha: ${{ fromJson(steps.pull_request_data.outputs.data).head.sha }} + target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + description: "vulnerability scan running" + state: "pending" + context: "blossom-ci" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} + ref: ${{ fromJson(steps.pull_request_data.outputs.data).head.ref }} + lfs: 'true' + + - name: Setup java + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Get project data (maven) + run: | + echo ::set-env name=projects::$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g') + + - name: Add mask + run: echo "::add-mask::${{ secrets.BLACKDUCK_URL }}" + + - name: Run synopsys detect + id: scan_result + uses: blackducksoftware/github-action@2.0.1 + env: + PROJECTS: ${{ env.projects }} + with: + args: > + --blackduck.url="https://${{ secrets.BLACKDUCK_URL }}" + --blackduck.api.token="${{ secrets.BLACKDUCK_API_TOKEN }}" + --detect.maven.build.command="-pl='$PROJECTS -am'" + --detect.force.success=false + --detect.parallel.processors=0 + --detect.project.name="${{ github.repository }}" + --detect.project.version.name="${{ github.run_id }}" + + vulnerability-check-trigger: + name: Vulnerability check & start ci job + needs: [vulnerability-scan-job] + runs-on: [self-hosted, linux, blossom] + steps: + - name: Check for new issue in vulnerability scan & start ci job + run: blossom-ci + env: + OPERATION: 'SCAN-CHECK-CI-JOB-START' + VERSION: '1' + BLACKDUCK_TOKEN: "${{ secrets.BLACKDUCK_API_TOKEN }}" + BLACKDUCK_URL: "${{ secrets.BLACKDUCK_URL }}" + BLACKDUCK_PROJECT_VERSION: "${{ github.run_id }}" + CI_SERVER: ${{ secrets.CI_SERVER }} + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/jenkins/Dockerfile-blossom.ubuntu16 b/jenkins/Dockerfile-blossom.ubuntu16 new file mode 100644 index 00000000000..73d33b2e9f8 --- /dev/null +++ b/jenkins/Dockerfile-blossom.ubuntu16 @@ -0,0 +1,40 @@ +# +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### +# +# Build the image for rapids-plugin development environment +# +# Arguments: CUDA_VER=10.1 or 10.2 +# +### + +ARG CUDA_VER=10.1 + +FROM nvidia/cuda:${CUDA_VER}-runtime-ubuntu16.04 + +#Install java-8, maven, docker image +RUN apt-get update -y && \ + apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update -y && \ + apt-get install -y maven \ + openjdk-8-jdk python3.6 python3-pip tzdata git + +RUN ln -s /usr/bin/python3.6 /usr/bin/python +RUN python -m pip install pytest sre_yield requests pandas pyarrow + +RUN apt install -y inetutils-ping expect diff --git a/jenkins/Jenkinsfile-blossom.premerge b/jenkins/Jenkinsfile-blossom.premerge new file mode 100644 index 00000000000..7f36c3f3e04 --- /dev/null +++ b/jenkins/Jenkinsfile-blossom.premerge @@ -0,0 +1,277 @@ +#!/usr/local/env groovy +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Jenkinsfile for building rapids-plugin on blossom + * + */ + +@Library('shared-libs') _ +@Library('blossom-github-lib@master') +import ipp.blossom.* + +def pluginDockerBuild = """ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: plugin-docker + image: docker:19.03.12 + command: + - sleep + args: + - 99d + resources: + requests: + cpu: 4 + memory: 8Gi + limits: + cpu: 8 + memory: 16Gi + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: plugin-docker-daemon + image: docker:19.03.12-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" + nodeSelector: + kubernetes.io/os: linux +""" + +def pluginPremerge = """ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: plugin-premerge + image: IMAGE_PREMERGE + imagePullPolicy: Always + resources: + requests: + cpu: 8 + memory: 16Gi + limits: + cpu: 16 + memory: 32Gi + nvidia.com/gpu: 1 + restartPolicy: Never + backoffLimit: 2 + tty: true + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu_type + operator: In + values: + - Tesla_V100S_PCIE_32GB + - Tesla_V100_PCIE_32GB + - TITAN_RTX + - Tesla_T4 +""" + +def githubHelper // blossom github helper +def IMAGE_PREMERGE // temp image for premerge test +def PREMERGE_TAG +def skipped = false + +pipeline { + agent { + kubernetes { + label "premerge-init-${BUILD_TAG}" + cloud 'sc-ipp-blossom-prod' + } + } + + options { + ansiColor('xterm') + buildDiscarder(logRotator(numToKeepStr: '50')) + skipDefaultCheckout true + timeout(time: 180, unit: 'MINUTES') + } + + environment { + JENKINS_ROOT = 'jenkins' + PREMERGE_SCRIPT = '$JENKINS_ROOT/spark-premerge-build.sh' + MVN_URM_MIRROR = '-s jenkins/settings.xml -P mirror-apache-to-urm' + LIBCUDF_KERNEL_CACHE_PATH = '/tmp/.cudf' + ARTIFACTORY_NAME = "${ArtifactoryConstants.ARTIFACTORY_NAME}" + GITHUB_TOKEN = credentials("github-token") + // TODO: rename this credential after we shutdown premerge pipeline on ngcc + URM_CREDS = credentials("svcngcc_artifactory") + URM_URL = "https://${ArtifactoryConstants.ARTIFACTORY_NAME}/artifactory/sw-spark-maven" + } + + stages { + stage("Init githubHelper") { + steps { + script { + githubHelper = GithubHelper.getInstance("${GITHUB_TOKEN}", githubData) + def title = githubHelper.getIssue().title + + if (title ==~ /.*\[skip ci\].*/) { + githubHelper.updateCommitStatus("$BUILD_URL", "Skipped", GitHubCommitState.SUCCESS) + currentBuild.result == "SUCCESS" + skipped = true + return + } + } + } + } // end of Init githubHelper + + stage('Build docker image') { + when { + beforeAgent true + expression { + !skipped + } + } + + agent { + kubernetes { + label "premerge-docker-${BUILD_TAG}" + cloud 'sc-ipp-blossom-prod' + yaml "$pluginDockerBuild" + } + } + + steps { + script { + githubHelper.updateCommitStatus("$BUILD_URL", "Running", GitHubCommitState.PENDING) + + currentBuild.description = githubHelper.getBuildDescription() + checkout( + changelog: false, + poll: true, + scm: [ + $class: 'GitSCM', branches: [[name: "pr/" + githubHelper.getPRNumber()]], + doGenerateSubmoduleConfigurations: false, + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'github-token', + url: githubHelper.getCloneUrl(), + refspec: '+refs/pull/*/head:refs/remotes/origin/pr/*']] + ] + ) + + container('plugin-docker') { + def CUDA_NAME = sh(returnStdout: true, + script: '. jenkins/version-def.sh>&2 && echo -n $CUDA_CLASSIFIER | sed "s/-/./g"') + + IMAGE_TAG = "dev-ubuntu16-${CUDA_NAME}" + CACHE_IMAGE_NAME = "${ARTIFACTORY_NAME}/sw-spark-docker/plugin:${IMAGE_TAG}" + + sh "docker pull $CACHE_IMAGE_NAME || true" + + PREMERGE_TAG = "${IMAGE_TAG}-${BUILD_TAG}" + IMAGE_PREMERGE = "${ARTIFACTORY_NAME}/sw-spark-docker-local/plugin:${PREMERGE_TAG}" + def CUDA_VER = "$CUDA_NAME" - "cuda" + docker.build(IMAGE_PREMERGE, "-f jenkins/Dockerfile-blossom.ubuntu16 --build-arg CUDA_VER=$CUDA_VER --cache-from $CACHE_IMAGE_NAME -t $IMAGE_PREMERGE .") + uploadDocker(IMAGE_PREMERGE) + + pluginPremerge = pluginPremerge.replace("IMAGE_PREMERGE", "$IMAGE_PREMERGE") + } + } + } + } // end of Build docker image + + // TODO: support parallel testing for different spark versions + stage('Premerge Test') { + when { + beforeAgent true + expression { + !skipped + } + } + + agent { + kubernetes { + label "premerge-test-${BUILD_TAG}" + cloud 'sc-ipp-blossom-prod' + yaml "$pluginPremerge" + } + } + + steps { + script { + checkout( + changelog: false, + poll: true, + scm: [ + $class: 'GitSCM', branches: [[name: "pr/" + githubHelper.getPRNumber()]], + doGenerateSubmoduleConfigurations: false, + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'github-token', + url: githubHelper.getCloneUrl(), + refspec: '+refs/pull/*/head:refs/remotes/origin/pr/*']] + ] + ) + + container('plugin-premerge') { + sh "$PREMERGE_SCRIPT" + step([$class : 'JacocoPublisher', + execPattern : '**/target/jacoco.exec', + classPattern : 'target/jacoco_classes/', + sourcePattern : 'sql-plugin/src/main/java/,sql-plugin/src/main/scala/,shuffle-plugin/src/main/scala/', + sourceInclusionPattern: '**/*.java,**/*.scala' + ]) + } + } + } + } // end of Premerge Test + } // end of stages + + post { + always { + script { + if (skipped) { + return + } + + deleteDockerTempTag(PREMERGE_TAG) // clean premerge temp image + if (currentBuild.currentResult == "SUCCESS") { + githubHelper.updateCommitStatus("$BUILD_URL", "Success", GitHubCommitState.SUCCESS) + } else { + githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE) + } + } + } + } + +} // end of pipeline + +void uploadDocker(String IMAGE_NAME) { + def DOCKER_CMD = "docker --config $WORKSPACE/.docker" + sh """ + echo $URM_CREDS_PSW | $DOCKER_CMD login $ARTIFACTORY_NAME -u $URM_CREDS_USR --password-stdin + $DOCKER_CMD push $IMAGE_NAME + $DOCKER_CMD logout $ARTIFACTORY_NAME + """ +} + +void deleteDockerTempTag(String tag) { + sh "curl -u $URM_CREDS_USR:$URM_CREDS_PSW -XDELETE https://${ARTIFACTORY_NAME}/artifactory/sw-spark-docker-local/plugin/${tag} || true" +}