From ab029f1fbce89ca0ac72252d5478f4e077a9dcb6 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 25 Apr 2024 18:25:22 +0300 Subject: [PATCH 1/8] Overwrite the branch for packaging Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/dra.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 4957ef36199e..1893674d7fcb 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -3,7 +3,7 @@ ## TODO: Set to empty string when Jenkins is disabled if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi set -euo pipefail -BRANCH="${BUILDKITE_BRANCH}" +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then BRANCH=main @@ -29,6 +29,7 @@ echo "+++ Changing permissions for the BK API commands" sudo chown -R :1000 build/distributions/ echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -43,8 +44,10 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" +set -x echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -60,3 +63,4 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ ${DRY_RUN} +set -x From e0927c2aeba1610cd284d3e423d018899242eaee Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 25 Apr 2024 18:30:19 +0300 Subject: [PATCH 2/8] Remove chown workaround Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/dra.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 1893674d7fcb..684b46e622e6 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -25,8 +25,6 @@ function release_manager_login { release_manager_login -echo "+++ Changing permissions for the BK API commands" -sudo chown -R :1000 build/distributions/ echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." set +x From 28d46b9263b351c8ccb4de7bedbe3893865ae1a3 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 25 Apr 2024 20:38:06 +0300 Subject: [PATCH 3/8] Change chmod Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/dra.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 684b46e622e6..2fccbe17cb03 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -25,9 +25,11 @@ function release_manager_login { release_manager_login +chmod -R a+r build/* +chmod -R a+w build echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." -set +x +set -x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -42,10 +44,10 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" -set -x +set +x echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." -set +x +set -x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -61,4 +63,4 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ ${DRY_RUN} -set -x +set +x From 2e354cd79b49c72407cc553c4c56251b4523bec1 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 26 Apr 2024 10:09:13 +0300 Subject: [PATCH 4/8] Various fixes --- .buildkite/scripts/dra.sh | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 2fccbe17cb03..e76e5e0686f9 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,8 +1,28 @@ #!/usr/bin/env bash -## TODO: Set to empty string when Jenkins is disabled -if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi +# TODO: uncomment out below when Jenkins packaging has been stopped +# if [[ "$DRY_RUN" == "true" ]]; then +# echo "~~~ Running in dry-run mode -- will NOT publish artifacts" +# DRY_RUN="--dry-run" +# else +# echo "~~~ Running in publish mode" +# DRY_RUN="" +# fi + + +# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped +if [[ "$DRY_RUN" == "false" ]]; then + echo "~~~ Running in publish mode" + DRY_RUN="" +else + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" + DRY_RUN="--dry-run" +fi + set -euo pipefail + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then @@ -25,11 +45,12 @@ function release_manager_login { release_manager_login +# required by the release-manager docker image, otherwise we hit: +# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) chmod -R a+r build/* chmod -R a+w build -echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." -set -x +echo "+++ :clipboard: Listing DRA artifacts for branch: $BRANCH using workflow: $DRA_WORKFLOW" docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -44,10 +65,8 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" -set +x -echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." -set -x +echo "+++ :hammer_and_pick: Publishing DRA artifacts for branch: $BRANCH using workflow: $DRA_WORKFLOW" docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -63,4 +82,3 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ ${DRY_RUN} -set +x From ee7c4167433564e65e8d8ec25452c281bdf8148c Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 26 Apr 2024 10:43:18 +0300 Subject: [PATCH 5/8] better debug --- .buildkite/scripts/dra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index e76e5e0686f9..f23d836a114b 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -50,7 +50,7 @@ release_manager_login chmod -R a+r build/* chmod -R a+w build -echo "+++ :clipboard: Listing DRA artifacts for branch: $BRANCH using workflow: $DRA_WORKFLOW" +echo "+++ :clipboard: Listing DRA artifacts for branch [$BRANCH] using workflow[$DRA_WORKFLOW]" docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -66,7 +66,7 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" -echo "+++ :hammer_and_pick: Publishing DRA artifacts for branch: $BRANCH using workflow: $DRA_WORKFLOW" +echo "+++ :hammer_and_pick: Publishing DRA artifacts for branch [$BRANCH] using workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ From bcccc3ce64182bd2a452101a9d9d64d33e9fcda1 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 26 Apr 2024 10:56:43 +0300 Subject: [PATCH 6/8] make scripts more defensive in terms of security --- .buildkite/scripts/dra.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index f23d836a114b..85a5a89fbaf8 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -9,7 +9,6 @@ # DRY_RUN="" # fi - # TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped if [[ "$DRY_RUN" == "false" ]]; then echo "~~~ Running in publish mode" @@ -43,14 +42,17 @@ function release_manager_login { export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET } +set +x release_manager_login + # required by the release-manager docker image, otherwise we hit: # > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) chmod -R a+r build/* chmod -R a+w build -echo "+++ :clipboard: Listing DRA artifacts for branch [$BRANCH] using workflow[$DRA_WORKFLOW]" +echo "+++ :clipboard: Listing DRA artifacts for branch [$BRANCH] using workflow [$DRA_WORKFLOW]" +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -67,6 +69,8 @@ docker run --rm \ --artifact-set "main" echo "+++ :hammer_and_pick: Publishing DRA artifacts for branch [$BRANCH] using workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" + +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ From 9d2942099fbf41a8a68686cb8d1338213322facc Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 26 Apr 2024 11:12:38 +0300 Subject: [PATCH 7/8] final fixes --- .buildkite/scripts/dra.sh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 85a5a89fbaf8..d905b0c2cbd3 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -24,12 +24,6 @@ set -euo pipefail # e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then - BRANCH=main - DRY_RUN="--dry-run" - echo "+++ Running in PR and setting branch main and --dry-run" -fi - BEAT_VERSION=$(make get-version) CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" @@ -45,13 +39,12 @@ function release_manager_login { set +x release_manager_login - # required by the release-manager docker image, otherwise we hit: # > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) chmod -R a+r build/* chmod -R a+w build -echo "+++ :clipboard: Listing DRA artifacts for branch [$BRANCH] using workflow [$DRA_WORKFLOW]" +echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]" set +x docker run --rm \ --name release-manager \ @@ -68,7 +61,7 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" -echo "+++ :hammer_and_pick: Publishing DRA artifacts for branch [$BRANCH] using workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" +echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" set +x docker run --rm \ From 277fab49fb9fd070f378c2cc106966616cc83953 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 26 Apr 2024 11:16:11 +0300 Subject: [PATCH 8/8] add summary annotation --- .buildkite/scripts/dra.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index d905b0c2cbd3..b20d6c6a696b 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -78,4 +78,13 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} + ${DRY_RUN} | tee rm-output.txt + +# extract the summary URL from a release manager output line like: +# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + +SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') +rm rm-output.txt + +# and make it easily clickable as a Builkite annotation +printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success