Merge pull request #97 from buildkite/pdp-1449-makebuildkite-agent-sc…

…aler-be-published-to-s3-again

Create a new release process

.buildkite/lib/release-dry-run.sh

@@ -0,0 +1,7 @@
+release_dry_run() {
+  if [[ "${RELEASE_DRY_RUN:-false}" == "true" ]]; then
+    echo "$@"
+  else
+    "$@"
+  fi
+}

.buildkite/pipeline-sar.yml → .buildkite/pipeline-sar.yaml

@@ -1,35 +1,31 @@
 steps:
-  - name: ":golang:"
+  - name: ":golang: Tests"
     key: test
-    command: ".buildkite/steps/tests.sh"
+    command: .buildkite/steps/tests.sh
     plugins:
       - docker#v3.1.0:
           image: "golang:1.19"
           workdir: /go/src/github.com/buildkite/buildkite-agent-scaler
 
-  - name: ":hammer: :lambda:"
+  - name: ":hammer::lambda: Build Lambda"
     key: build
     command: .buildkite/steps/build-lambda.sh
     artifact_paths:
       - handler.zip
 
-  - label: ":lambda: :arrow_right: :package:"
+  - label: ":lambda::arrow_right::package: Create SAR Package"
     key: package
-    command:
-      - buildkite-agent artifact download handler.zip .
-      - sam package --region us-east-1 --s3-bucket buildkite-serverless-apps-us-east-1 --s3-prefix elastic-ci/agent-scaler --output-template-file packaged.yml
-      - buildkite-agent artifact upload 'packaged.yml'
-      - echo "--- Template"; echo "$(< packaged.yml)"
+    command: .buildkite/steps/sar.sh
     depends_on:
       - test
       - build
 
-  - block: ":package: :arrow_right: :rocket:?"
+  - block: ":package::arrow_right::rocket: Draft :github: Release Approved?"
     key: release
     depends_on:
       - package
 
-  - label: ":package: :arrow_right: :aws:"
+  - label: ":package::arrow_right::aws: Publish SAR Package"
     key: publish
     command: .buildkite/steps/publish-package.sh
     depends_on:

.buildkite/pipeline.yaml

@@ -0,0 +1,74 @@
+agents:
+  queue: elastic-runners
+
+steps:
+- name: ":go: Check go fmt"
+  key: go-fmt
+  command: .buildkite/steps/test-go-fmt.sh
+  plugins:
+  - docker#v5.8.0:
+      image: golang:1.19
+      workdir: /go/src/github.com/buildkite/buildkite-agent-scaler
+
+- name: ":golang: Run Tests"
+  key: test
+  command: .buildkite/steps/tests.sh
+  plugins:
+  - docker#v5.8.0:
+      image: golang:1.19
+      workdir: /go/src/github.com/buildkite/buildkite-agent-scaler
+
+- name: ":lambda: Build Lambda"
+  key: build-lambda
+  depends_on:
+  - go-fmt
+  command: .buildkite/steps/build-lambda.sh
+  artifact_paths:
+  - handler.zip
+
+- label: ":s3: Pubish to S3 Branch Location"
+  key: s3-branch
+  depends_on:
+  - test
+  - build-lambda
+  command: .buildkite/steps/upload-to-s3.sh
+  plugins:
+  - aws-assume-role-with-web-identity:
+      role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-scaler
+
+- if: build.tag =~ /^.+\$/ || build.env("RELEASE_DRY_RUN") == "true"
+  label: ":github: Draft GitHub Release"
+  key: github-release
+  depends_on:
+  - build-lambda
+  command: .buildkite/steps/github-release.sh
+  env:
+    BUILDKITE_AGENT_GIT_FETCH_FLAGS: -v --prune --tags
+  plugins:
+  - aws-assume-role-with-web-identity:
+      role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-buildkite-agent-scaler
+  - aws-ssm#v1.0.0:
+      parameters:
+        GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-scaler/GITHUB_RELEASE_ACCESS_TOKEN
+  - docker#v5.8.0:
+      image: alpine:3.18
+      propagate-environment: true
+      mount-buildkite-agent: true
+      environment:
+      - GITHUB_RELEASE_ACCESS_TOKEN
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+
+- if: build.tag =~ /^.+\$/ || build.env("RELEASE_DRY_RUN") == "true"
+  key: block-s3-release
+  block: ":rocket: Draft :github: Release Approved?"
+
+- if: build.tag =~ /^.+\$/ || build.env("RELEASE_DRY_RUN") == "true"
+  label: ":s3: Publish to S3 Release Location"
+  key: s3-release
+  depends_on:
+  - github-release
+  - block-s3-release
+  command: .buildkite/steps/upload-to-s3.sh release
+  plugins:
+  - aws-assume-role-with-web-identity:
+      role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-scaler

.buildkite/steps/github-release.sh

@@ -1,27 +1,54 @@
-#!/usr/bin/env bash
+#!/usr/bin/env ash
 
 set -eufo pipefail
 
-echo '--- Getting credentials from SSM'
-GITHUB_RELEASE_ACCESS_TOKEN=$( \
-  aws ssm get-parameter \
-    --name /pipelines/buildkite-agent-scaler/GITHUB_RELEASE_ACCESS_TOKEN \
-    --with-decryption \
-    --output text \
-    --query Parameter.Value \
-    --region us-east-1 \
-)
-
-if [[ "$GITHUB_RELEASE_ACCESS_TOKEN" == "" ]]; then
-  echo "Error: Missing \$GITHUB_RELEASE_ACCESS_TOKEN"
+source .buildkite/lib/release-dry-run.sh
+
+if [[ "${RELEASE_DRY_RUN:-false}" != true && -z "$BUILDKITE_TAG" ]]; then
+  echo ^^^ +++
+  echo This step should only be run on a tag >&2
+  exit 1
+fi
+
+echo --- :hammer: Installing packages
+apk add --no-progress git github-cli
+
+echo --- Checking tags
+version=$(awk -F\" '/const Version/ {print $2}' version/version.go)
+tag="v${version#v}"
+
+if [[ "${RELEASE_DRY_RUN:-false}" != true && "$tag" != "$BUILDKITE_TAG" ]]; then
+  echo ^^^ +++
+  echo "Error: version.go has not been updated to ${BUILDKITE_TAG#v}" >&2
   exit 1
 fi
 
-VERSION=$(buildkite-agent meta-data get "version")
-buildkite-agent artifact download "handler.zip" .
+last_tag=$(git describe --tags --abbrev=0 --exclude "$tag")
 
-echo "--- 🚀 Releasing $VERSION"
-export GITHUB_RELEASE_ACCESS_TOKEN
-github-release "v${VERSION#v}" handler.zip \
-  --commit "$(git rev-parse HEAD)" \
-  --github-repository "buildkite/buildkite-agent-scaler"
+# escape . so we can use in regex
+escaped_tag="${tag//\./\\.}"
+escaped_last_tag="${last_tag//\./\\.}"
+
+echo --- The following notes will accompany the release:
+# The sed commands below:
+#   Find lines between headers of the changelogs (inclusive)
+#   Delete the lines included from the headers
+# The command substituion will then delete the empty lines from the end
+notes=$(sed -n "/^## \[${escaped_tag}\]/,/^## \[${escaped_last_tag}\]/p" CHANGELOG.md | sed '$d')
+echo "$notes"
+
+echo --- :lambda: Downloading lambda from artifacts
+buildkite-agent artifact download handler.zip .
+
+echo "--- 🚀 Releasing $version"
+if [[ "${GITHUB_RELEASE_ACCESS_TOKEN:-}" == "" ]]; then
+  echo ^^^ +++
+  echo "Error: Missing \$GITHUB_RELEASE_ACCESS_TOKEN"
+  exit 1
+fi
+GITHUB_TOKEN="$GITHUB_RELEASE_ACCESS_TOKEN" release_dry_run gh release create \
+  --draft \
+  --title "$tag" \
+  --notes "$notes" \
+  "$tag" \
+  handler.zip

.buildkite/steps/publish-package.sh

@@ -2,18 +2,20 @@
 
 set -eufo pipefail
 
+source .buildkite/lib/release-dry-run.sh
+
 VERSION=$(buildkite-agent meta-data get version)
 
-buildkite-agent artifact download 'packaged.yml' .
+buildkite-agent artifact download packaged.yml .
 
-if [[ -z $BUILDKITE_TAG ]]; then
+if [[ -z "${BUILDKITE_TAG:-}" ]]; then
   APP_ID=arn:aws:serverlessrepo:us-east-1:172840064832:applications/buildkite-agent-scaler-edge
 else
   APP_ID=arn:aws:serverlessrepo:us-east-1:172840064832:applications/buildkite-agent-scaler
 fi
 
 echo --- ":aws::lambda: Publishing version $VERSION to SAR"
-aws serverlessrepo create-application-version \
+release_dry_run aws serverlessrepo create-application-version \
   --application-id "$APP_ID" \
   --template-body file://packaged.yml \
   --semantic-version "${VERSION#v}" \

.buildkite/steps/sar.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo --- Download handler.zip
+buildkite-agent artifact download handler.zip .
+
+echo --- Create template for Serverless Application Repository
+sam package --region us-east-1 --s3-bucket buildkite-serverless-apps-us-east-1 --s3-prefix elastic-ci/agent-scaler --output-template-file packaged.yml
+buildkite-agent artifact upload packaged.yml
+
+echo --- Print template for Serverless Application Repository
+echo "$(< packaged.yml)"

.buildkite/steps/upload-to-s3.sh

@@ -1,9 +1,8 @@
-#!/bin/bash
-set -eu
+#!/usr/bin/env bash
+
+set -euo pipefail
 
 export AWS_DEFAULT_REGION=us-east-1
-# export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-$SANDBOX_AWS_ACCESS_KEY_ID}
-# export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-$SANDBOX_AWS_SECRET_ACCESS_KEY}
 
 EXTRA_REGIONS=(
   us-east-2
@@ -27,24 +26,24 @@ EXTRA_REGIONS=(
   sa-east-1
 )
 
-VERSION=$(buildkite-agent meta-data get "version")
+VERSION=$(buildkite-agent meta-data get version)
 BASE_BUCKET=buildkite-lambdas
-BUCKET_PATH="buildkite-agent-scaler"
+BUCKET_PATH=buildkite-agent-scaler
 
-if [[ "${1:-}" == "release" ]] ; then
-  BUCKET_PATH="${BUCKET_PATH}/v${VERSION}"
+if [[ "${1:-}" == "release" ]]; then
+  BUCKET_PATH="${BUCKET_PATH}/${VERSION}"
 else
   BUCKET_PATH="${BUCKET_PATH}/builds/${BUILDKITE_BUILD_NUMBER}"
 fi
 
 echo "~~~ :buildkite: Downloading artifacts"
 buildkite-agent artifact download handler.zip .
 
-echo "--- :s3: Uploading lambda to ${BASE_BUCKET}/${BUCKET_PATH}/ in ${AWS_DEFAULT_REGION}"
+echo "~~~ :s3: Uploading lambda to ${BASE_BUCKET}/${BUCKET_PATH}/ in ${AWS_DEFAULT_REGION}"
 aws s3 cp --acl public-read handler.zip "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip"
 
-for region in "${EXTRA_REGIONS[@]}" ; do
-	bucket="${BASE_BUCKET}-${region}"
-	echo "--- :s3: Copying files to ${bucket}"
-	aws --region "${region}" s3 cp --acl public-read "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip" "s3://${bucket}/${BUCKET_PATH}/handler.zip"
+for region in "${EXTRA_REGIONS[@]}"; do
+  bucket="${BASE_BUCKET}-${region}"
+  echo "~~~ :s3: Copying files to ${bucket}"
+  aws --region "${region}" s3 cp --acl public-read "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip" "s3://${bucket}/${BUCKET_PATH}/handler.zip"
 done