Merge pull request #82 from buildkite/pdp-777-fix-ci-for-buildkite-ag…

…ent-scaler
buildkite · Apr 2, 2023 · ebf0d22 · ebf0d22
2 parents b918626 + 8a74c83
commit ebf0d22
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 40 deletions.
diff --git a/.buildkite/pipeline.release.yml b/.buildkite/pipeline.release.yml
@@ -5,7 +5,7 @@ steps:
     command: ".buildkite/steps/release-version.sh"
     branches: master
     agents:
-      queue: "deploy"
+      queue: elastic-runners
     concurrency: 1
     concurrency_group: 'release'
 
@@ -14,13 +14,18 @@ steps:
     command: ".buildkite/steps/github-release.sh"
     branches: master
     agents:
-      queue: "deploy"
+      queue: elastic-runners
     concurrency: 1
     concurrency_group: 'release'
     plugins:
+      - aws-assume-role-with-web-identity:
+          role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-agent-scaler
+      - aws-ssm#v1.0.0:
+          parameters:
+            GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite-agent-scaler/GITHUB_RELEASE_ACCESS_TOKEN
       - ecr#v2.0.0:
           login: true
-          account-ids: "032379705303"
+          account-ids: "445615400570"
       - docker#v3.5.0:
-          image: "032379705303.dkr.ecr.us-east-1.amazonaws.com/deploytools:2020.03"
+          image: "445615400570.dkr.ecr.us-east-1.amazonaws.com/deploytools:2020.03"
           propagate-environment: true
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -16,7 +16,10 @@ steps:
   - label: ":s3:"
     command: ".buildkite/steps/upload-to-s3.sh"
     agents:
-      queue: "deploy"
+      queue: elastic-runners
+    plugins:
+      - aws-assume-role-with-web-identity:
+          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-scaler
 
   - wait
   - name: ":pipeline:"

diff --git a/lambda/main.go b/lambda/main.go
@@ -27,8 +27,8 @@ var (
 
 type LastScaleASGResult struct {
 	LastScaleOutActivity *autoscaling.Activity
-	LastScaleInActivity *autoscaling.Activity
-	Err error
+	LastScaleInActivity  *autoscaling.Activity
+	Err                  error
 }
 
 func main() {
@@ -50,7 +50,7 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 		interval time.Duration    = 10 * time.Second
 
 		asgActivityTimeoutDuration = 10 * time.Second
-		asgActivityTimeout = time.After(asgActivityTimeoutDuration)
+		asgActivityTimeout         = time.After(asgActivityTimeoutDuration)
 
 		scaleInCooldownPeriod time.Duration
 		scaleInFactor         float64
@@ -164,19 +164,19 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 	scalingLastActivityStartTime := time.Now()
 	go func() {
 		scaleOutOutput, scaleInOutput, err := asg.GetLastScalingInAndOutActivity()
-		result := LastScaleASGResult {
-			scaleOutOutput, 
+		result := LastScaleASGResult{
+			scaleOutOutput,
 			scaleInOutput,
 			err,
 		}
 		c1 <- result
 	}()
 
 	select {
-    case res := <-c1:
+	case res := <-c1:
 		if res.Err != nil {
 			log.Printf("Failed to retrieve last scaling activity events due to error (%s)", res.Err)
-			break;
+			break
 		}
 
 		scaleInOutput := res.LastScaleInActivity
@@ -189,8 +189,8 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 		}
 		scalingTimeDiff := time.Now().Sub(scalingLastActivityStartTime)
 		log.Printf("Succesfully retrieved last scaling activity events. Last scale out %v, last scale in %v. Discovery took %s.", lastScaleOut, lastScaleIn, scalingTimeDiff)
-    case <- asgActivityTimeout:
-        log.Printf("Failed to retrieve last scaling activity events due to %s timeout", asgActivityTimeoutDuration)
+	case <-asgActivityTimeout:
+		log.Printf("Failed to retrieve last scaling activity events due to %s timeout", asgActivityTimeoutDuration)
 	}
 
 	for {
@@ -231,7 +231,7 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 					Factor:         scaleOutFactor,
 					LastEvent:      lastScaleOut,
 				},
-				InstanceBuffer: instanceBuffer,
+				InstanceBuffer:         instanceBuffer,
 				ScaleOnlyAfterAllEvent: scaleOnlyAfterAllEvent,
 			}
 

diff --git a/scaler/asg.go b/scaler/asg.go
@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	activitySucessfulStatusCode = "Successful"
+	activitySucessfulStatusCode           = "Successful"
 	userRequestForChangingDesiredCapacity = "a user request explicitly set group desired capacity changing the desired capacity"
 )
 
@@ -89,7 +89,7 @@ func (a *ASGDriver) GetAutoscalingActivities(nextToken *string) (*autoscaling.De
 	svc := autoscaling.New(a.Sess)
 	input := &autoscaling.DescribeScalingActivitiesInput{
 		AutoScalingGroupName: aws.String(a.Name),
-		NextToken: nextToken,
+		NextToken:            nextToken,
 	}
 	return svc.DescribeScalingActivities(input)
 }

diff --git a/scaler/scaler.go b/scaler/scaler.go
@@ -42,10 +42,10 @@ type Scaler struct {
 	metrics interface {
 		Publish(orgSlug, queue string, metrics map[string]int64) error
 	}
-	scaling        ScalingCalculator
-	scaleInParams  ScaleParams
-	scaleOutParams ScaleParams
-	instanceBuffer int
+	scaling                ScalingCalculator
+	scaleInParams          ScaleParams
+	scaleOutParams         ScaleParams
+	instanceBuffer         int
 	scaleOnlyAfterAllEvent bool
 }
 
@@ -55,9 +55,9 @@ func NewScaler(client *buildkite.Client, sess *session.Session, params Params) (
 			client: client,
 			queue:  params.BuildkiteQueue,
 		},
-		scaleInParams:  params.ScaleInParams,
-		scaleOutParams: params.ScaleOutParams,
-		instanceBuffer: params.InstanceBuffer,
+		scaleInParams:          params.ScaleInParams,
+		scaleOutParams:         params.ScaleOutParams,
+		instanceBuffer:         params.InstanceBuffer,
 		scaleOnlyAfterAllEvent: params.ScaleOnlyAfterAllEvent,
 	}
 

diff --git a/scaler/scaler_test.go b/scaler/scaler_test.go
@@ -232,7 +232,7 @@ func TestScalingOutWithoutError(t *testing.T) {
 		{
 			metrics: buildkite.AgentMetrics{
 				ScheduledJobs: 1,
-				IdleAgents:  0,
+				IdleAgents:    0,
 				TotalAgents:   0,
 			},
 			params: Params{
@@ -242,7 +242,7 @@ func TestScalingOutWithoutError(t *testing.T) {
 					CooldownPeriod: 2 * time.Minute,
 				},
 				ScaleInParams: ScaleParams{
-					LastEvent:time.Now().Add(-1 * time.Minute),
+					LastEvent:      time.Now().Add(-1 * time.Minute),
 					CooldownPeriod: 5 * time.Minute,
 				},
 				ScaleOnlyAfterAllEvent: true,
@@ -256,11 +256,11 @@ func TestScalingOutWithoutError(t *testing.T) {
 				desiredCapacity: tc.currentDesiredCapacity,
 			}
 			s := Scaler{
-				autoscaling:    asg,
-				bk:             &buildkiteTestDriver{metrics: tc.metrics},
-				scaleOutParams: tc.params.ScaleOutParams,
-				scaleInParams: tc.params.ScaleInParams,
-				instanceBuffer: tc.params.InstanceBuffer,
+				autoscaling:            asg,
+				bk:                     &buildkiteTestDriver{metrics: tc.metrics},
+				scaleOutParams:         tc.params.ScaleOutParams,
+				scaleInParams:          tc.params.ScaleInParams,
+				instanceBuffer:         tc.params.InstanceBuffer,
 				scaleOnlyAfterAllEvent: tc.params.ScaleOnlyAfterAllEvent,
 				scaling: ScalingCalculator{
 					includeWaiting:    tc.params.IncludeWaiting,
@@ -330,7 +330,7 @@ func TestScalingInWithoutError(t *testing.T) {
 			},
 			params: Params{
 				AgentsPerInstance: 1,
-				InstanceBuffer: 10,
+				InstanceBuffer:    10,
 			},
 			currentDesiredCapacity:  30,
 			expectedDesiredCapacity: 25,
@@ -354,8 +354,8 @@ func TestScalingInWithoutError(t *testing.T) {
 		// Make sure we round down so we eventually reach zero
 		{
 			metrics: buildkite.AgentMetrics{
-				IdleAgents: 1,
-				TotalAgents:   1,
+				IdleAgents:  1,
+				TotalAgents: 1,
 			},
 			params: Params{
 				AgentsPerInstance: 1,
@@ -369,7 +369,7 @@ func TestScalingInWithoutError(t *testing.T) {
 		// Scale in disabled
 		{
 			metrics: buildkite.AgentMetrics{
-				TotalAgents:   1,
+				TotalAgents: 1,
 			},
 			params: Params{
 				ScaleInParams: ScaleParams{
@@ -383,7 +383,7 @@ func TestScalingInWithoutError(t *testing.T) {
 		{
 			metrics: buildkite.AgentMetrics{
 				IdleAgents:  3,
-				TotalAgents:   3,
+				TotalAgents: 3,
 			},
 			params: Params{
 				AgentsPerInstance: 1,
@@ -392,7 +392,7 @@ func TestScalingInWithoutError(t *testing.T) {
 					CooldownPeriod: 5 * time.Minute,
 				},
 				ScaleInParams: ScaleParams{
-					LastEvent:time.Now().Add(-10 * time.Minute),
+					LastEvent:      time.Now().Add(-10 * time.Minute),
 					CooldownPeriod: 2 * time.Minute,
 				},
 				ScaleOnlyAfterAllEvent: true,
@@ -414,9 +414,9 @@ func TestScalingInWithoutError(t *testing.T) {
 					includeWaiting:    tc.params.IncludeWaiting,
 					agentsPerInstance: tc.params.AgentsPerInstance,
 				},
-				scaleInParams: tc.params.ScaleInParams,
-				scaleOutParams: tc.params.ScaleOutParams,
-				instanceBuffer: tc.params.InstanceBuffer,
+				scaleInParams:          tc.params.ScaleInParams,
+				scaleOutParams:         tc.params.ScaleOutParams,
+				instanceBuffer:         tc.params.InstanceBuffer,
 				scaleOnlyAfterAllEvent: tc.params.ScaleOnlyAfterAllEvent,
 			}