From e09b6a85a2f89c9516ec56205c47391d36ee2eca Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 22 Mar 2022 10:09:33 +0000 Subject: [PATCH] ci: support arn automation within release pipeline (#2453) --- .ci/Makefile | 88 +++++++++++++++++++++++++++++++++++++++++ .ci/create-arn-table.sh | 26 ++++++++++++ .ci/release/Jenkinsfile | 52 +++++++++++++++++++++++- .gitignore | 1 + 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 .ci/Makefile create mode 100755 .ci/create-arn-table.sh diff --git a/.ci/Makefile b/.ci/Makefile new file mode 100644 index 0000000000..036d12e71f --- /dev/null +++ b/.ci/Makefile @@ -0,0 +1,88 @@ +SHELL = /bin/bash -eo pipefail + +AWS_FOLDER = .aws +AWS_LAMBDA_ZIP_LOCATION = elastic-apm-agent/target + +export AWS_FOLDER + +dist: + @cp ../$(AWS_LAMBDA_ZIP_LOCATION)/elastic-apm-java-aws-lambda-layer-*.zip ../$(AWS_LAMBDA_ZIP_LOCATION)/$(ELASTIC_LAYER_NAME).zip + +# List all the AWS regions +get-all-aws-regions: + @aws \ + ec2 \ + describe-regions \ + --region us-east-1 \ + --output json \ + --no-cli-pager \ + | jq -r '.Regions[].RegionName' > .regions + +# Publish the given LAYER in all the AWS regions +publish-in-all-aws-regions: validate-layer-name get-all-aws-regions dist + @mkdir -p $(AWS_FOLDER) + @while read AWS_DEFAULT_REGION; do \ + echo "publish '$(ELASTIC_LAYER_NAME)' in $${AWS_DEFAULT_REGION}"; \ + AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish > $(AWS_FOLDER)/$${AWS_DEFAULT_REGION}; \ + AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) grant-public-layer-access; \ + done <.regions + +# Publish the given LAYER in the given AWS region +publish: validate-layer-name validate-aws-default-region + @aws lambda \ + --output json \ + publish-layer-version \ + --layer-name "$(ELASTIC_LAYER_NAME)" \ + --description "AWS Lambda Extension Layer for the Elastic APM Java Agent" \ + --compatible-runtimes java8.al2 java11 \ + --license "Apache-2.0" \ + --zip-file "fileb://../$(AWS_LAMBDA_ZIP_LOCATION)/$(ELASTIC_LAYER_NAME).zip" + +# Grant public access to the given LAYER in the given AWS region +grant-public-layer-access: validate-layer-name validate-aws-default-region + @echo "[debug] $(ELASTIC_LAYER_NAME) with version: $$($(MAKE) -s --no-print-directory get-version)" + @aws lambda \ + --output json \ + add-layer-version-permission \ + --layer-name "$(ELASTIC_LAYER_NAME)" \ + --action lambda:GetLayerVersion \ + --principal '*' \ + --statement-id "$(ELASTIC_LAYER_NAME)" \ + --version-number $$($(MAKE) -s --no-print-directory get-version) > $(AWS_FOLDER)/.$(AWS_DEFAULT_REGION)-public + +# Generate the file with the ARN entries +create-arn-file: validate-suffix-arn-file + @./create-arn-table.sh + +upload-lambda-asset: validate-release-version validate-layer-name + @gh release list + @gh \ + release \ + upload $(RELEASE_VERSION) \ + ./$(AWS_LAMBDA_ZIP_LOCATION)/$(ELASTIC_LAYER_NAME).zip + +# Get the ARN Version for the AWS_REGIONS +# NOTE: jq -r .Version "$(AWS_FOLDER)/$(AWS_DEFAULT_REGION)" fails in the CI +# with 'parse error: Invalid numeric literal at line 1, column 5' +get-version: validate-aws-default-region + @grep '"Version"' "$(AWS_FOLDER)/$(AWS_DEFAULT_REGION)" | cut -d":" -f2 | sed 's/ //g' | cut -d"," -f1 + +validate-release-version: +ifndef RELEASE_VERSION + $(error RELEASE_VERSION is undefined) +endif + +validate-suffix-arn-file: +ifndef SUFFIX_ARN_FILE + $(error SUFFIX_ARN_FILE is undefined) +endif + +validate-layer-name: +ifndef ELASTIC_LAYER_NAME + $(error ELASTIC_LAYER_NAME is undefined) +endif + +validate-aws-default-region: +ifndef AWS_DEFAULT_REGION + $(error AWS_DEFAULT_REGION is undefined) +endif diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh new file mode 100755 index 0000000000..fb89f1cc90 --- /dev/null +++ b/.ci/create-arn-table.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -o pipefail + +# +# Create the AWS ARN table given the below environment variables: +# +# - AWS_FOLDER - that's the location of the publish-layer-version output for each region +# - SUFFIX_ARN_FILE - that's the output file. +# + +ARN_FILE=${SUFFIX_ARN_FILE} + +{ + echo "### ARNs of the APM Java Agent's AWS Lambda Layer" + echo '' + echo '|Region|ARN|' + echo '|------|---|' +} > "${ARN_FILE}" + +for f in $(ls "${AWS_FOLDER}"); do + LAYER_VERSION_ARN=$(grep '"LayerVersionArn"' "$AWS_FOLDER/${f}" | cut -d":" -f2- | sed 's/ //g' | sed 's/"//g' | cut -d"," -f1) + echo "INFO: create-arn-table ARN(${LAYER_VERSION_ARN}):region(${f}))" + echo "|${f}|${LAYER_VERSION_ARN}|" >> "${ARN_FILE}" +done + +echo '' >> "${ARN_FILE}" diff --git a/.ci/release/Jenkinsfile b/.ci/release/Jenkinsfile index 1000f42ddc..601562c83c 100644 --- a/.ci/release/Jenkinsfile +++ b/.ci/release/Jenkinsfile @@ -16,6 +16,9 @@ pipeline { SLACK_CHANNEL = '#apm-agent-java' NOTIFY_TO = 'build-apm+apm-agent-java@elastic.co' BRANCH_SPECIFIER = "${params.branch_specifier}" + SUFFIX_ARN_FILE = 'arn-file.md' + //RELEASE_AWS_LAMBDA_VERSION = '-ver-1-29-0' + //RELEASE_VERSION = '1.29.0' } options { timeout(time: 3, unit: 'HOURS') @@ -27,6 +30,7 @@ pipeline { parameters { string(name: 'branch_specifier', defaultValue: 'stable', description: "What branch to release from?") booleanParam(name: 'check_branch_ci_status', defaultValue: true, description: "Check for failing tests in the given branch (if no stable branch)?") + booleanParam(name: 'publish_aws_lambda', defaultValue: true, description: "Whether to upload the AWS lambda") } stages { stage('Initializing'){ @@ -131,6 +135,7 @@ pipeline { env.RELEASE_TAG = "v" + user_release_version env.RELEASE_VERSION = user_release_version env.BRANCH_DOT_X = user_release_version.substring(0, user_release_version.indexOf('.'))+'.x' + env.RELEASE_AWS_LAMBDA_VERSION = '-ver-' + user_release_version.replaceAll('\\.', '-') } } } @@ -164,10 +169,43 @@ pipeline { } } } + stage('Publish AWS Lambda') { + when { + //expression { params.publish_aws_lambda } + expression { return false } + } + environment { + SOURCE_AWS_FILE = "elastic-apm-java-aws-lambda-layer-${RELEASE_VERSION}.zip" + } + steps { + setEnvVar('ELASTIC_LAYER_NAME', "elastic-apm-java${env.RELEASE_AWS_LAMBDA_VERSION}") + withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', role_id: 'apm-vault-role-id', secret_id: 'apm-vault-secret-id', + forceInstallation: true, version: '2.4.10') { + dir("${BASE_DIR}"){ + dir ('elastic-apm-agent/target') { + // TODO: copy file from a google bucket + sh(label: 'fetch AWS lambda file', script: "wget https://github.com/elastic/apm-agent-java/releases/download/v${RELEASE_VERSION}/${SOURCE_AWS_FILE} -O ${SOURCE_AWS_FILE}") + } + sh(label: 'make publish-in-all-aws-regions', script: 'make -C .ci publish-in-all-aws-regions') + sh(label: 'make create-arn-file', script: 'make -C .ci create-arn-file') + } + } + } + post { + always { + archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/.ci/${SUFFIX_ARN_FILE}") + } + } + } stage('Create GitHub release draft') { + when { + expression { return false } + } steps { dir("${BASE_DIR}"){ script { + def arnFile = ".ci/${SUFFIX_ARN_FILE}" + setEnvVar('ARN_CONTENT', fileExists(arnFile) ? readFile(arnFile) : '') // Construct the URL with anchor for the release notes // Ex: https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-1.x.html#release-notes-1.13.0 def finalUrl = "https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${BRANCH_DOT_X}.html#release-notes-${RELEASE_VERSION}" @@ -176,7 +214,7 @@ pipeline { draft: true, tagName: "${RELEASE_TAG}", releaseName: "Release ${RELEASE_VERSION}", - body: "[Release Notes for ${RELEASE_VERSION}](${finalUrl})") + body: "[Release Notes for ${RELEASE_VERSION}](${finalUrl}) \n ${ARN_CONTENT}") env.RELEASE_ID = ret['id'] env.RELEASE_NOTES_URL = finalUrl } @@ -184,6 +222,9 @@ pipeline { } } stage('Wait for artifact to be available in Maven Central') { + when { + expression { return false } + } steps { dir("${BASE_DIR}"){ script { @@ -199,6 +240,9 @@ pipeline { } } stage('Update Cloudfoundry') { + when { + expression { return false } + } steps { dir("${BASE_DIR}"){ sh(script: ".ci/release/update_cloudfoundry.sh ${RELEASE_VERSION}") @@ -207,6 +251,9 @@ pipeline { } } stage('Publish release on GitHub') { + when { + expression { return false } + } steps { dir("${BASE_DIR}"){ waitUntil(initialRecurrencePeriod: 60000) { @@ -222,6 +269,9 @@ pipeline { } } stage('Build and push Docker images') { + when { + expression { return false } + } steps { dir("${BASE_DIR}"){ // fetch agent artifact from remote repository diff --git a/.gitignore b/.gitignore index 51ef5a2a8f..f211042b10 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ docs/html/ **/__pycache__/ .m2/ fetch.log +.ci/.aws