From 1fb3ec2071330afaad1cb9f7c0bd057adab3a0cc Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 15 Oct 2021 17:56:22 -0700 Subject: [PATCH 1/5] Add workflow to stage maven release Signed-off-by: Sayali Gaikawad --- jenkins/stage-maven-release/JenkinsFile | 102 ++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 jenkins/stage-maven-release/JenkinsFile diff --git a/jenkins/stage-maven-release/JenkinsFile b/jenkins/stage-maven-release/JenkinsFile new file mode 100644 index 0000000000..e184fae658 --- /dev/null +++ b/jenkins/stage-maven-release/JenkinsFile @@ -0,0 +1,102 @@ +pipeline { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:al2-x64-arm64-jdk14-node10.24.1-cypress6.9.1-20211005' + // Unlike freestyle docker, pipeline docker does not login to the container and run commands + // It use executes which does not source the docker container internal ENV VAR + args '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot' + alwaysPull true + } + } + environment { + ARTIFACT_PATH = "/usr/share/opensearch/.m2/repository/org/opensearch/client/opensearch-java/${VERSION}" + VERSION = "${params.VERSION}" + } + stages { + stage('parameters') { + steps { + script { + properties([ + parameters([ + string( + defaultValue: '', + name: 'REF', + trim: true + ), + string( + name: 'VERSION', + trim: true + ) + ]) + ]) + } + } + } + stage('Publish to maven local') { + steps { + // checkout the commit + git url: 'https://github.com/opensearch-project/opensearch-java.git', branch: 'main' + sh('git checkout ${REF}') + + //publish to maven local + sh('./gradlew publishtoMavenLocal') + sh('ls -l ${ARTIFACT_PATH}') + } + } + stage('Sign the artifacts') { + environment { + // These ENV variables are required by https://github.com/opensearch-project/opensearch-signer-client + // This client is invoked internally by the sign script. + ROLE = "${SIGNER_CLIENT_ROLE}" + EXTERNAL_ID = "${SIGNER_CLIENT_EXTERNAL_ID}" + UNSIGNED_BUCKET = "${SIGNER_CLIENT_UNSIGNED_BUCKET}" + SIGNED_BUCKET = "${SIGNER_CLIENT_SIGNED_BUCKET}" + } + steps { + // Fetch opensearch public key and add to keyring. + sh('curl https://artifacts.opensearch.org/publickeys/opensearch.pgp -o $WORKSPACE/opensearch.pgp') + sh('gpg --import $WORKSPACE/opensearch.pgp') + + // Sign artifacts + git credentialsId: 'jenkins-staging-github-bot-token', + url: 'https://github.com/opensearch-project/opensearch-signer-client.git', branch: 'main' + dir("src"){ + sh ('./bootstrap') + sh('rm config.cfg') + sh('ls -d ${ARTIFACT_PATH}/* | xargs -I {} sh -c \'./opensearch-signer-client -i {} -o {}.sig -p pgp\'') + + // Transform the binary signature to an ascii armored file + sh("for i in `ls -d ${ARTIFACT_PATH}/*.sig`; do (cat \$i | gpg --enarmor | sed 's/ARMORED FILE/SIGNATURE/g') > \${i%%.sig}.asc; done") + + // Verify they are ANSI with PGP SIGNATURE + sh('ls -d ${ARTIFACT_PATH}/*.asc | xargs -I {} sh -c \'cat {} | grep PGP\'') + + // Verify the signatures + sh('ls -d ${ARTIFACT_PATH}/*.asc | xargs -I {} sh -c \'gpg --verify {} \'') + + // Remove sig files + sh('rm -f ${ARTIFACT_PATH}/*.sig') + } + } + } + stage('Stage maven artifacts') { + tools { + maven "maven-3.8.2" + } + environment { + REPO_URL = "https://aws.oss.sonatype.org/" + STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}" + BUILD_ID = "${BUILD_NUMBER}" + } + steps { + // checkout the build repo + git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + // stage artifacts for release with Sonatype + withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + sh('$WORKSPACE/publish/stage-maven-release.sh /usr/share/opensearch/.m2/repository/') + } + } + } + } +} \ No newline at end of file From 351fb068106d7987f50e8ff6ed52e58b57dddac6 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 15 Oct 2021 23:08:56 -0700 Subject: [PATCH 2/5] Fixes Signed-off-by: Sayali Gaikawad --- jenkins/stage-maven-release/JenkinsFile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/jenkins/stage-maven-release/JenkinsFile b/jenkins/stage-maven-release/JenkinsFile index e184fae658..95f1c2c6f6 100644 --- a/jenkins/stage-maven-release/JenkinsFile +++ b/jenkins/stage-maven-release/JenkinsFile @@ -90,11 +90,20 @@ pipeline { BUILD_ID = "${BUILD_NUMBER}" } steps { + // copy artifacts to other folder so that not everything from .m2 gets staged + dir("$OUTPUT_DIR"){ + sh('cp -r /usr/share/opensearch/.m2/repository/org/opensearch/ $OUTPUT_DIR/org/') + sh('bash -O extglob -c "rm -rf $OUTPUT_DIR/org/opensearch/!(client)"') + sh('ls -l $OUTPUT_DIR/org/') + + } + // checkout the build repo git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + // stage artifacts for release with Sonatype withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { - sh('$WORKSPACE/publish/stage-maven-release.sh /usr/share/opensearch/.m2/repository/') + sh('$WORKSPACE/publish/stage-maven-release.sh $OUTPUT_DIR') } } } From f9c412a04767f1d4cefa5e9183a87ad2c327d845 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 15 Oct 2021 23:14:52 -0700 Subject: [PATCH 3/5] Fixes Signed-off-by: Sayali Gaikawad --- jenkins/stage-maven-release/JenkinsFile | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/stage-maven-release/JenkinsFile b/jenkins/stage-maven-release/JenkinsFile index 95f1c2c6f6..b0a7cdba1f 100644 --- a/jenkins/stage-maven-release/JenkinsFile +++ b/jenkins/stage-maven-release/JenkinsFile @@ -85,6 +85,7 @@ pipeline { maven "maven-3.8.2" } environment { + OUTPUT_DIR = "$WORKSPACE/maven-signed" REPO_URL = "https://aws.oss.sonatype.org/" STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}" BUILD_ID = "${BUILD_NUMBER}" From 2e3b8495b55f792db61f738131157ce9455f5612 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 18 Oct 2021 13:26:54 -0700 Subject: [PATCH 4/5] Param check Signed-off-by: Sayali Gaikawad --- jenkins/stage-maven-release/JenkinsFile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins/stage-maven-release/JenkinsFile b/jenkins/stage-maven-release/JenkinsFile index b0a7cdba1f..88aeb37e7e 100644 --- a/jenkins/stage-maven-release/JenkinsFile +++ b/jenkins/stage-maven-release/JenkinsFile @@ -30,6 +30,10 @@ pipeline { ) ]) ]) + if (params.REF.isEmpty() || params.VERSION.isEmpty()) { + currentBuild.result = 'ABORTED' + error('One or both of the parameters is empty') + } } } } From ad902483f5c05b50674573a63f4952ecc082cdd7 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 18 Oct 2021 17:20:38 -0700 Subject: [PATCH 5/5] Add checksum Signed-off-by: Sayali Gaikawad --- jenkins/stage-maven-release/JenkinsFile | 47 +++++++++++++++++-------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/jenkins/stage-maven-release/JenkinsFile b/jenkins/stage-maven-release/JenkinsFile index 88aeb37e7e..269967b958 100644 --- a/jenkins/stage-maven-release/JenkinsFile +++ b/jenkins/stage-maven-release/JenkinsFile @@ -11,6 +11,7 @@ pipeline { } environment { ARTIFACT_PATH = "/usr/share/opensearch/.m2/repository/org/opensearch/client/opensearch-java/${VERSION}" + OUTPUT_DIR = "$WORKSPACE/maven-signed" VERSION = "${params.VERSION}" } stages { @@ -33,11 +34,11 @@ pipeline { if (params.REF.isEmpty() || params.VERSION.isEmpty()) { currentBuild.result = 'ABORTED' error('One or both of the parameters is empty') - } + } } } } - stage('Publish to maven local') { + stage('Publish to Maven Local') { steps { // checkout the commit git url: 'https://github.com/opensearch-project/opensearch-java.git', branch: 'main' @@ -45,10 +46,9 @@ pipeline { //publish to maven local sh('./gradlew publishtoMavenLocal') - sh('ls -l ${ARTIFACT_PATH}') } - } - stage('Sign the artifacts') { + } + stage('Sign') { environment { // These ENV variables are required by https://github.com/opensearch-project/opensearch-signer-client // This client is invoked internally by the sign script. @@ -84,25 +84,39 @@ pipeline { } } } + stage('Generate checksums') { + steps { + dir("$OUTPUT_DIR/org"){ + // copy only required artifacts to other folder so that not everything from .m2 gets staged + sh('cp -r /usr/share/opensearch/.m2/repository/org/opensearch $OUTPUT_DIR/org/') + sh('bash -O extglob -c "rm -rf $OUTPUT_DIR/org/opensearch/!(client)"') + sh ''' + for file in $(find \$OUTPUT_DIR/org/opensearch/client/opensearch-java/\${VERSION} -type f) + do + if [ \${file##*.} != "asc" ] + then + echo "Creating checksum for \$file" + (md5sum \$file | cut -d \' \' -f 1) > \$file.md5 + (sha1sum \$file | cut -d \' \' -f 1) > \$file.sha1 + (sha256sum \$file | cut -d \' \' -f 1) > \$file.sha256 + (sha512sum \$file | cut -d \' \' -f 1) > \$file.sha512 + fi + done + ''' + } + sh('ls -l \$OUTPUT_DIR/org/opensearch/client/opensearch-java/\${VERSION}') + } + } stage('Stage maven artifacts') { tools { maven "maven-3.8.2" } environment { - OUTPUT_DIR = "$WORKSPACE/maven-signed" REPO_URL = "https://aws.oss.sonatype.org/" STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}" BUILD_ID = "${BUILD_NUMBER}" } steps { - // copy artifacts to other folder so that not everything from .m2 gets staged - dir("$OUTPUT_DIR"){ - sh('cp -r /usr/share/opensearch/.m2/repository/org/opensearch/ $OUTPUT_DIR/org/') - sh('bash -O extglob -c "rm -rf $OUTPUT_DIR/org/opensearch/!(client)"') - sh('ls -l $OUTPUT_DIR/org/') - - } - // checkout the build repo git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' @@ -111,6 +125,11 @@ pipeline { sh('$WORKSPACE/publish/stage-maven-release.sh $OUTPUT_DIR') } } + post() { + always { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } } } } \ No newline at end of file