From a090ea70a0fd8aefd45220ac8fcadb38a4a5ec7e Mon Sep 17 00:00:00 2001 From: mitchya1 Date: Thu, 10 Jun 2021 20:30:33 -0600 Subject: [PATCH 1/5] WIP - add blue green support --- README.md | 10 ++- cmd/plugin/helpers.go | 75 +++++++++++++++++ cmd/plugin/main.go | 189 ++++++++++++++++++++++++++++++------------ go.mod | 1 + go.sum | 2 + pkg/deploy/service.go | 62 ++++++++++++++ pkg/types/types.go | 3 + 7 files changed, 290 insertions(+), 52 deletions(-) create mode 100644 cmd/plugin/helpers.go diff --git a/README.md b/README.md index c68a4cb..7b91504 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ steps: - name: deploy image: public.ecr.aws/assemblyai/drone-deploy-ecs settings: + # Can either be rolling or blue-green + mode: rolling aws_region: us-east-2 # The name of the ECS service service: webapp @@ -51,4 +53,10 @@ steps: # The image to deploy image: myorg/nginx-${DRONE_COMMIT_SHA} max_deploy_checks: 10 -``` \ No newline at end of file +``` + +## Blue / Green + +The ECS service must have an associated Application Autoscaling Target + +The deployment type must be ECS diff --git a/cmd/plugin/helpers.go b/cmd/plugin/helpers.go new file mode 100644 index 0000000..ecd80fe --- /dev/null +++ b/cmd/plugin/helpers.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "errors" + "log" + "os" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecs" +) + +func checkEnvVars() error { + requiredVars := []string{ + "PLUGIN_AWS_REGION", + "PLUGIN_SERVICE", + "PLUGIN_CLUSTER", + "PLUGIN_CONTAINER", + "PLUGIN_IMAGE", + "PLUGIN_MODE", + } + + for _, v := range requiredVars { + if os.Getenv(v) == "" { + log.Printf("Required environment variable '%s' is missing\n", v) + return errors.New("env var not set") + } + } + + return nil +} + +func parseRollingVars() error { + requiredVars := []string{ + "PLUGIN_SERVICE", + } + + for _, v := range requiredVars { + if os.Getenv(v) == "" { + log.Printf("Required environment variable '%s' is missing\n", v) + return errors.New("env var not set") + } + } + + return nil +} + +func checkBlueGreenVars() error { + requiredVars := []string{ + "PLUGIN_BLUE_SERVICE_NAME", + "PLUGIN_GREEN_SERVICE_NAME", + } + + for _, v := range requiredVars { + if os.Getenv(v) == "" { + log.Printf("Required environment variable '%s' is missing\n", v) + return errors.New("env var not set") + } + } + + return nil +} + +func newECSClient(region string) *ecs.Client { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + ) + + if err != nil { + log.Fatalf("Failed to load SDK configuration, %v", err) + } + + return ecs.NewFromConfig(cfg) +} diff --git a/cmd/plugin/main.go b/cmd/plugin/main.go index 7a997bd..77d86f8 100644 --- a/cmd/plugin/main.go +++ b/cmd/plugin/main.go @@ -10,44 +10,60 @@ import ( "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" "github.com/assemblyai/drone-deploy-ecs/pkg/types" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ecs" ) const ( defaultMaxChecksUntilFailed = 60 // 10 second between checks + 60 checks = 600 seconds = 10 minutes ) -func checkEnvVars() error { - requiredVars := []string{ - "PLUGIN_AWS_REGION", - "PLUGIN_SERVICE", - "PLUGIN_CLUSTER", - "PLUGIN_CONTAINER", - "PLUGIN_IMAGE", - } +/* +Blue / Green - for _, v := range requiredVars { - if os.Getenv(v) == "" { - log.Printf("Required environment variable '%s' is missing\n", v) - return errors.New("env var not set") - } - } +QUESTIONS: +- What do we do about autoscaling? + - Need to figure out how to set max count to 0 - return nil -} +When we do a blue/green deployment, we need to discover which service is "green". As far as the plugin is concerned, the service with 0 replicas is green + +If we can't decide which service is blue and which is green, we should exit with a reconcile error + +After we decide which service to update first, we need to modify the task definition. This is the same as a rolling deployment. + +Now that we know which service is blue, we need to figure out how many replicas it has. We'll set green to have the same + +After the task definition is updated, we need to set the green service to use the new TD version + +We'll continue to use the ECS service deployment status for deciding if the deployment is working or not + +Once the green service is healthy, we can scale down blue +*/ + +func determineBlueGreen(e types.ECSClient, blueService string, greenService string, cluster string) (string, string, error) { + blueCount, err := deploy.GetServiceDesiredCount(context.Background(), e, blueService, cluster) + + if err != nil { + log.Println("Error retrieving desired count for blue service", err.Error()) + return "", "", errors.New("deploy failed") + } -func newECSClient(region string) *ecs.Client { - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion(region), - ) + greenCount, err := deploy.GetServiceDesiredCount(context.Background(), e, greenService, cluster) if err != nil { - log.Fatalf("Failed to load SDK configuration, %v", err) + log.Println("Error retrieving desired count for blue service", err.Error()) + return "", "", errors.New("deploy failed") + } + + if blueCount == 0 { + return blueService, greenService, nil } - return ecs.NewFromConfig(cfg) + if greenCount == 0 { + return greenService, blueService, nil + } + + log.Println("Unable to determine which service is blue and which is green") + log.Printf("Service '%s' has %d desired replicas while service '%s' has %d desired replicas. One of these should be 0", blueService, blueCount, greenService, greenCount) + return "", "", errors.New("reconcile error") } // Return values -> success (bool), error @@ -105,53 +121,89 @@ func release(e types.ECSClient, service string, cluster string, maxDeployChecks return true, nil } -func main() { - // Ensure all required env vars are present - if err := checkEnvVars(); err != nil { - os.Exit(1) +func blueGreen(e types.ECSClient, cluster string, container string, image string) error { + blueServiceName := os.Getenv("PLUGIN_BLUE_SERVICE_NAME") + greenServiceName := os.Getenv("PLUGIN_GREEN_SERVICE_NAME") + + determinedBlueService, determinedGreenService, err := determineBlueGreen(e, blueServiceName, greenServiceName, cluster) + + if err != nil { + return err } - e := newECSClient(os.Getenv("PLUGIN_AWS_REGION")) + td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, determinedBlueService, cluster) - var maxDeployChecks int + if err != nil { + log.Println("Failing because of an error determining the currently in-use task definition") + return err + } - service := os.Getenv("PLUGIN_SERVICE") - cluster := os.Getenv("PLUGIN_CLUSTER") - container := os.Getenv("PLUGIN_CONTAINER") - image := os.Getenv("PLUGIN_IMAGE") + currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) - if os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS") == "" { - log.Println("PLUGIN_MAX_DEPLOY_CHECKS environment variable not set. Defaulting to", defaultMaxChecksUntilFailed) - maxDeployChecks = defaultMaxChecksUntilFailed - } else { - convertResult, err := strconv.Atoi(os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS")) - if err != nil { - log.Printf("Error converting '%s' to int. Defaulting to 60 checks, which is 10 minutes\n", os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS")) - maxDeployChecks = defaultMaxChecksUntilFailed - } else { - maxDeployChecks = convertResult - } + if err != nil { + log.Println("Failing because of an error retrieving the currently in-use task definition") + return err } + newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) + + if err != nil { + log.Println("Failing because of an error retrieving the creating a new task definition revision") + return err + } + + log.Println("Created new task definition revision", newTD.Revision) + + currBlueDesiredCount, err := deploy.GetServiceDesiredCount(context.Background(), e, determinedBlueService, cluster) + + if err != nil { + log.Println("Failing because of an error determining desired count for blue service", err.Error()) + return err + } + + // There is no deployment so discard it + _, err = deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, determinedGreenService, cluster, *newTD.TaskDefinitionArn) + + if err != nil { + log.Println("Error updating task definition for service", err.Error()) + return errors.New("deploy failed") + } + + // Scale up green service to the same count as blue + // TODO pass the actual max instead of 0 + deploy.ScaleUp(e, currBlueDesiredCount, 0) + log.Println("Pausing for 10 seconds while ECS schedules", currBlueDesiredCount, " containers") + time.Sleep(10 * time.Second) + + // Loop and check GreenScaleUpFinished - timeout and fail after maxDeployChecks + + // Scale down + + return nil +} + +func rolling(e types.ECSClient, cluster string, container string, image string, maxDeployChecks int) error { + service := os.Getenv("PLUGIN_SERVICE") + td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, service, cluster) if err != nil { log.Println("Failing because of an error determining the currently in-use task definition") - os.Exit(1) + return errors.New("deploy failed") } currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) if err != nil { log.Println("Failing because of an error retrieving the currently in-use task definition") - os.Exit(1) + return errors.New("deploy failed") } newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) if err != nil { log.Println("Failing because of an error retrieving the creating a new task definition revision") - os.Exit(1) + return errors.New("deploy failed") } log.Println("Created new task definition revision", newTD.Revision) @@ -165,10 +217,45 @@ func main() { if !rollbackOK { log.Println("Error rolling back") } - // Exit 1 so the build fails - os.Exit(1) + return errors.New("deploy failed") } log.Println("Deployment succeeded") + return nil +} +func main() { + // Ensure all required env vars are present + if err := checkEnvVars(); err != nil { + os.Exit(1) + } + + e := newECSClient(os.Getenv("PLUGIN_AWS_REGION")) + + var maxDeployChecks int + + if os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS") == "" { + log.Println("PLUGIN_MAX_DEPLOY_CHECKS environment variable not set. Defaulting to", defaultMaxChecksUntilFailed) + maxDeployChecks = defaultMaxChecksUntilFailed + } else { + convertResult, err := strconv.Atoi(os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS")) + if err != nil { + log.Printf("Error converting '%s' to int. Defaulting to 60 checks, which is 10 minutes\n", os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS")) + maxDeployChecks = defaultMaxChecksUntilFailed + } else { + maxDeployChecks = convertResult + } + } + + cluster := os.Getenv("PLUGIN_CLUSTER") + container := os.Getenv("PLUGIN_CONTAINER") + image := os.Getenv("PLUGIN_IMAGE") + + if os.Getenv("PLUGIN_MODE") == "blue-green" { + + } else { + if err := rolling(e, cluster, container, image, maxDeployChecks); err != nil { + os.Exit(1) + } + } } diff --git a/go.mod b/go.mod index 7a9ce27..eadc1fc 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/aws/aws-sdk-go-v2 v1.6.0 github.com/aws/aws-sdk-go-v2/config v1.3.0 + github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.3.1 // indirect github.com/aws/aws-sdk-go-v2/service/ecs v1.5.0 github.com/pkg/errors v0.9.1 // indirect gotest.tools v2.2.0+incompatible diff --git a/go.sum b/go.sum index a739733..cf77566 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1 h1:w1ocBIhQkLgupEB3d0uOuBdd github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1/go.mod h1:GTXAhrxHQOj9N+J5tYVjwt+rpRyy/42qLjlgw9pz1a0= github.com/aws/aws-sdk-go-v2/internal/ini v1.0.0 h1:k7I9E6tyVWBo7H9ffpnxDWudtjau6Qt9rnOYgV+ciEQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.0.0/go.mod h1:g3XMXuxvqSMUjnsXXp/960152w0wFS4CXVYgQaSVOHE= +github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.3.1 h1:26gxzxyog/sYh3l98BQ0F/SmXXtEwPe/7PWoBpMX9dE= +github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.3.1/go.mod h1:ofzhcv3bOqs4rHqLn9pZwk4LnkrJqgpxLxzgNQhUgBw= github.com/aws/aws-sdk-go-v2/service/ecs v1.5.0 h1:csM92rzQqRt8q/awf2TwI86Kqn9X3aEdo4RR36HhD9s= github.com/aws/aws-sdk-go-v2/service/ecs v1.5.0/go.mod h1:xqNV1bRJobA3QD870zptCh3t9uUbdWgReH+brKIt8As= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.1.1 h1:l7pDLsmOGrnR8LT+3gIv8NlHpUhs7220E457KEC2UM0= diff --git a/pkg/deploy/service.go b/pkg/deploy/service.go index 8f7d259..47ca00c 100644 --- a/pkg/deploy/service.go +++ b/pkg/deploy/service.go @@ -3,6 +3,7 @@ package deploy import ( "context" "errors" + "fmt" "log" "github.com/assemblyai/drone-deploy-ecs/pkg/types" @@ -29,6 +30,31 @@ func GetServiceRunningTaskDefinition(ctx context.Context, c types.ECSClient, ser return *out.Services[0].TaskDefinition, nil } +func GetServiceDesiredCount(ctx context.Context, c types.ECSClient, service string, cluster string) (int32, error) { + i := ecs.DescribeServicesInput{ + Services: []string{service}, + Cluster: aws.String(cluster), + } + + out, err := c.DescribeServices( + ctx, + &i, + ) + + if err != nil { + log.Println("Error describing service: ", err.Error()) + return 0, err + } + + return out.Services[0].DesiredCount, nil +} + +func GetServiceMaxCount(c types.AppAutoscalingClient, cluster string, service string) { + resourceID := fmt.Sprintf("service/%s/%s", cluster, service) + // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/applicationautoscaling#Client.DescribeScalableTargets + +} + func UpdateServiceTaskDefinitionVersion(ctx context.Context, c types.ECSClient, service string, cluster string, taskDefinitonARN string) (string, error) { i := ecs.UpdateServiceInput{ @@ -52,6 +78,7 @@ func UpdateServiceTaskDefinitionVersion(ctx context.Context, c types.ECSClient, } // CheckDeploymentStatus returns true if a deployment has finished (either success or failure) and false if the deployment is in progress +// TODO remove deploymentID func CheckDeploymentStatus(ctx context.Context, c types.ECSClient, service string, cluster string, deploymentID string) (bool, error) { i := ecs.DescribeServicesInput{ Services: []string{service}, @@ -78,3 +105,38 @@ func CheckDeploymentStatus(ctx context.Context, c types.ECSClient, service strin } } + +func GreenScaleUpFinished(ctx context.Context, c types.ECSClient, service string, cluster string) (bool, error) { + i := ecs.DescribeServicesInput{ + Services: []string{service}, + Cluster: aws.String(cluster), + } + + out, err := c.DescribeServices( + ctx, + &i, + ) + + if err != nil { + log.Println("Error describing service: ", err.Error()) + return true, err + } + + if out.Services[0].RunningCount != out.Services[0].DesiredCount { + return false, nil + } + + return true, nil +} + +func ScaleDown(c types.ECSClient, desiredCount int32, minCount int32, maxCount int32) { + // Set max, desired, min to 0 +} + +func ScaleUp(c types.ECSClient, count int32, maxCount int32) { + // Set max count to maxCount +} + +func GetContainerStatus() { + +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 23d6c2f..854129b 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -12,3 +12,6 @@ type ECSClient interface { DescribeServices(ctx context.Context, params *ecs.DescribeServicesInput, optFns ...func(*ecs.Options)) (*ecs.DescribeServicesOutput, error) UpdateService(ctx context.Context, params *ecs.UpdateServiceInput, optFns ...func(*ecs.Options)) (*ecs.UpdateServiceOutput, error) } + +type AppAutoscalingClient interface { +} From e84a29d97adebc1fc3a75aa3a50ba58f043fd958 Mon Sep 17 00:00:00 2001 From: mitchya1 Date: Fri, 11 Jun 2021 07:20:40 -0600 Subject: [PATCH 2/5] WIP - cleanup, add application autoscaling --- cmd/plugin/blue_green.go | 127 +++++++++++++++++++++++ cmd/plugin/helpers.go | 14 +++ cmd/plugin/main.go | 214 --------------------------------------- cmd/plugin/rolling.go | 109 ++++++++++++++++++++ pkg/deploy/scaling.go | 73 +++++++++++++ pkg/deploy/service.go | 19 ---- pkg/types/types.go | 3 + 7 files changed, 326 insertions(+), 233 deletions(-) create mode 100644 cmd/plugin/blue_green.go create mode 100644 cmd/plugin/rolling.go create mode 100644 pkg/deploy/scaling.go diff --git a/cmd/plugin/blue_green.go b/cmd/plugin/blue_green.go new file mode 100644 index 0000000..a37b739 --- /dev/null +++ b/cmd/plugin/blue_green.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "errors" + "log" + "os" + "time" + + "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" + "github.com/assemblyai/drone-deploy-ecs/pkg/types" +) + +/* +Blue / Green + +When we do a blue/green deployment, we need to discover which service is "green". As far as the plugin is concerned, the service with 0 replicas is green + +If we can't decide which service is blue and which is green, we should exit with a reconcile error + +After we decide which service to update first, we need to modify the task definition. This is the same as a rolling deployment. + +Now that we know which service is blue, we need to figure out how many replicas it has. We'll set green to have the same + +After the task definition is updated, we need to set the green service to use the new TD version + +Once green is using new task definition, we need to check blue for an app autoscaling target. If it exists, +we need to set green to have the same max. Otherwise we'll just work off desired count + +Once we've done that, set green's desired count to blue's and watch until green's running count == desired count. This step will need to +have a timeout on it. If we reach the timeout, scale green back down and fail the deployment + +Once green is up and running, scale down blue. If there's an autoscaling target, scale down by decrementing the max, otherwise just +work off of the desired count + + +Once the green service is healthy, we can scale down blue +*/ + +func determineBlueGreen(e types.ECSClient, blueService string, greenService string, cluster string) (string, string, error) { + blueCount, err := deploy.GetServiceDesiredCount(context.Background(), e, blueService, cluster) + + if err != nil { + log.Println("Error retrieving desired count for blue service", err.Error()) + return "", "", errors.New("deploy failed") + } + + greenCount, err := deploy.GetServiceDesiredCount(context.Background(), e, greenService, cluster) + + if err != nil { + log.Println("Error retrieving desired count for blue service", err.Error()) + return "", "", errors.New("deploy failed") + } + + if blueCount == 0 { + return blueService, greenService, nil + } + + if greenCount == 0 { + return greenService, blueService, nil + } + + log.Println("Unable to determine which service is blue and which is green") + log.Printf("Service '%s' has %d desired replicas while service '%s' has %d desired replicas. One of these should be 0", blueService, blueCount, greenService, greenCount) + return "", "", errors.New("reconcile error") +} + +func blueGreen(e types.ECSClient, cluster string, container string, image string) error { + blueServiceName := os.Getenv("PLUGIN_BLUE_SERVICE_NAME") + greenServiceName := os.Getenv("PLUGIN_GREEN_SERVICE_NAME") + + determinedBlueService, determinedGreenService, err := determineBlueGreen(e, blueServiceName, greenServiceName, cluster) + + if err != nil { + return err + } + + td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, determinedBlueService, cluster) + + if err != nil { + log.Println("Failing because of an error determining the currently in-use task definition") + return err + } + + currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) + + if err != nil { + log.Println("Failing because of an error retrieving the currently in-use task definition") + return err + } + + newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) + + if err != nil { + log.Println("Failing because of an error retrieving the creating a new task definition revision") + return err + } + + log.Println("Created new task definition revision", newTD.Revision) + + currBlueDesiredCount, err := deploy.GetServiceDesiredCount(context.Background(), e, determinedBlueService, cluster) + + if err != nil { + log.Println("Failing because of an error determining desired count for blue service", err.Error()) + return err + } + + // There is no deployment so discard it + _, err = deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, determinedGreenService, cluster, *newTD.TaskDefinitionArn) + + if err != nil { + log.Println("Error updating task definition for service", err.Error()) + return errors.New("deploy failed") + } + + // Scale up green service to the same count as blue + // TODO pass the actual max instead of 0 + deploy.ScaleUp(e, currBlueDesiredCount, 0) + log.Println("Pausing for 10 seconds while ECS schedules", currBlueDesiredCount, " containers") + time.Sleep(10 * time.Second) + + // Loop and check GreenScaleUpFinished - timeout and fail after maxDeployChecks + + // Scale down + + return nil +} diff --git a/cmd/plugin/helpers.go b/cmd/plugin/helpers.go index ecd80fe..a7a9193 100644 --- a/cmd/plugin/helpers.go +++ b/cmd/plugin/helpers.go @@ -7,6 +7,7 @@ import ( "os" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/ecs" ) @@ -73,3 +74,16 @@ func newECSClient(region string) *ecs.Client { return ecs.NewFromConfig(cfg) } + +func newAppAutoscalingTarget(region string) *applicationautoscaling.Client { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + ) + + if err != nil { + log.Fatalf("Failed to load SDK configuration, %v", err) + } + + return applicationautoscaling.NewFromConfig(cfg) +} diff --git a/cmd/plugin/main.go b/cmd/plugin/main.go index 77d86f8..2883de4 100644 --- a/cmd/plugin/main.go +++ b/cmd/plugin/main.go @@ -1,229 +1,15 @@ package main import ( - "context" - "errors" "log" "os" "strconv" - "time" - - "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" - "github.com/assemblyai/drone-deploy-ecs/pkg/types" ) const ( defaultMaxChecksUntilFailed = 60 // 10 second between checks + 60 checks = 600 seconds = 10 minutes ) -/* -Blue / Green - -QUESTIONS: -- What do we do about autoscaling? - - Need to figure out how to set max count to 0 - -When we do a blue/green deployment, we need to discover which service is "green". As far as the plugin is concerned, the service with 0 replicas is green - -If we can't decide which service is blue and which is green, we should exit with a reconcile error - -After we decide which service to update first, we need to modify the task definition. This is the same as a rolling deployment. - -Now that we know which service is blue, we need to figure out how many replicas it has. We'll set green to have the same - -After the task definition is updated, we need to set the green service to use the new TD version - -We'll continue to use the ECS service deployment status for deciding if the deployment is working or not - -Once the green service is healthy, we can scale down blue -*/ - -func determineBlueGreen(e types.ECSClient, blueService string, greenService string, cluster string) (string, string, error) { - blueCount, err := deploy.GetServiceDesiredCount(context.Background(), e, blueService, cluster) - - if err != nil { - log.Println("Error retrieving desired count for blue service", err.Error()) - return "", "", errors.New("deploy failed") - } - - greenCount, err := deploy.GetServiceDesiredCount(context.Background(), e, greenService, cluster) - - if err != nil { - log.Println("Error retrieving desired count for blue service", err.Error()) - return "", "", errors.New("deploy failed") - } - - if blueCount == 0 { - return blueService, greenService, nil - } - - if greenCount == 0 { - return greenService, blueService, nil - } - - log.Println("Unable to determine which service is blue and which is green") - log.Printf("Service '%s' has %d desired replicas while service '%s' has %d desired replicas. One of these should be 0", blueService, blueCount, greenService, greenCount) - return "", "", errors.New("reconcile error") -} - -// Return values -> success (bool), error -func release(e types.ECSClient, service string, cluster string, maxDeployChecks int, taskDefinitionARN string) (bool, error) { - var err error - - deployCounter := 0 - deployFinished := false - deployFailed := false - - deploymentID, err := deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, service, cluster, taskDefinitionARN) - - if err != nil { - log.Println("Error updating task definition for service", err.Error()) - return true, errors.New("deploy failed") - } - - log.Println("Started deployment with ID", deploymentID) - - for !deployFinished { - // Ensure that we haven't hit this limit - // We want to rollback quickly - if deployCounter > maxDeployChecks { - log.Println("Reached max check limit. Will attempt rollback") - deployFinished = true - deployFailed = true - break - } - - log.Println("Waiting for deployment to complete. Check number:", deployCounter) - time.Sleep(10 * time.Second) - deployCounter++ - - deployFinished, err = deploy.CheckDeploymentStatus( - context.TODO(), - e, - service, - cluster, - deploymentID, - ) - - if err != nil { - log.Println("Deployment failed: ", err.Error()) - deployFinished = true - deployFailed = true - break - } - - } - - if deployFailed { - return false, errors.New("deploy failed") - } - - return true, nil -} - -func blueGreen(e types.ECSClient, cluster string, container string, image string) error { - blueServiceName := os.Getenv("PLUGIN_BLUE_SERVICE_NAME") - greenServiceName := os.Getenv("PLUGIN_GREEN_SERVICE_NAME") - - determinedBlueService, determinedGreenService, err := determineBlueGreen(e, blueServiceName, greenServiceName, cluster) - - if err != nil { - return err - } - - td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, determinedBlueService, cluster) - - if err != nil { - log.Println("Failing because of an error determining the currently in-use task definition") - return err - } - - currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) - - if err != nil { - log.Println("Failing because of an error retrieving the currently in-use task definition") - return err - } - - newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) - - if err != nil { - log.Println("Failing because of an error retrieving the creating a new task definition revision") - return err - } - - log.Println("Created new task definition revision", newTD.Revision) - - currBlueDesiredCount, err := deploy.GetServiceDesiredCount(context.Background(), e, determinedBlueService, cluster) - - if err != nil { - log.Println("Failing because of an error determining desired count for blue service", err.Error()) - return err - } - - // There is no deployment so discard it - _, err = deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, determinedGreenService, cluster, *newTD.TaskDefinitionArn) - - if err != nil { - log.Println("Error updating task definition for service", err.Error()) - return errors.New("deploy failed") - } - - // Scale up green service to the same count as blue - // TODO pass the actual max instead of 0 - deploy.ScaleUp(e, currBlueDesiredCount, 0) - log.Println("Pausing for 10 seconds while ECS schedules", currBlueDesiredCount, " containers") - time.Sleep(10 * time.Second) - - // Loop and check GreenScaleUpFinished - timeout and fail after maxDeployChecks - - // Scale down - - return nil -} - -func rolling(e types.ECSClient, cluster string, container string, image string, maxDeployChecks int) error { - service := os.Getenv("PLUGIN_SERVICE") - - td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, service, cluster) - - if err != nil { - log.Println("Failing because of an error determining the currently in-use task definition") - return errors.New("deploy failed") - } - - currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) - - if err != nil { - log.Println("Failing because of an error retrieving the currently in-use task definition") - return errors.New("deploy failed") - } - - newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) - - if err != nil { - log.Println("Failing because of an error retrieving the creating a new task definition revision") - return errors.New("deploy failed") - } - - log.Println("Created new task definition revision", newTD.Revision) - - deploymentOK, _ := release(e, service, cluster, maxDeployChecks, *newTD.TaskDefinitionArn) - - if !deploymentOK { - log.Println("Rolling back failed deployment") - rollbackOK, _ := release(e, service, cluster, maxDeployChecks, *currTD.TaskDefinitionArn) - - if !rollbackOK { - log.Println("Error rolling back") - } - return errors.New("deploy failed") - } - - log.Println("Deployment succeeded") - return nil -} - func main() { // Ensure all required env vars are present if err := checkEnvVars(); err != nil { diff --git a/cmd/plugin/rolling.go b/cmd/plugin/rolling.go new file mode 100644 index 0000000..95b75ef --- /dev/null +++ b/cmd/plugin/rolling.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "errors" + "log" + "os" + "time" + + "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" + "github.com/assemblyai/drone-deploy-ecs/pkg/types" +) + +// Return values -> success (bool), error +func release(e types.ECSClient, service string, cluster string, maxDeployChecks int, taskDefinitionARN string) (bool, error) { + var err error + + deployCounter := 0 + deployFinished := false + deployFailed := false + + deploymentID, err := deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, service, cluster, taskDefinitionARN) + + if err != nil { + log.Println("Error updating task definition for service", err.Error()) + return true, errors.New("deploy failed") + } + + log.Println("Started deployment with ID", deploymentID) + + for !deployFinished { + // Ensure that we haven't hit this limit + // We want to rollback quickly + if deployCounter > maxDeployChecks { + log.Println("Reached max check limit. Will attempt rollback") + deployFinished = true + deployFailed = true + break + } + + log.Println("Waiting for deployment to complete. Check number:", deployCounter) + time.Sleep(10 * time.Second) + deployCounter++ + + deployFinished, err = deploy.CheckDeploymentStatus( + context.TODO(), + e, + service, + cluster, + deploymentID, + ) + + if err != nil { + log.Println("Deployment failed: ", err.Error()) + deployFinished = true + deployFailed = true + break + } + + } + + if deployFailed { + return false, errors.New("deploy failed") + } + + return true, nil +} + +func rolling(e types.ECSClient, cluster string, container string, image string, maxDeployChecks int) error { + service := os.Getenv("PLUGIN_SERVICE") + + td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, service, cluster) + + if err != nil { + log.Println("Failing because of an error determining the currently in-use task definition") + return errors.New("deploy failed") + } + + currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) + + if err != nil { + log.Println("Failing because of an error retrieving the currently in-use task definition") + return errors.New("deploy failed") + } + + newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) + + if err != nil { + log.Println("Failing because of an error retrieving the creating a new task definition revision") + return errors.New("deploy failed") + } + + log.Println("Created new task definition revision", newTD.Revision) + + deploymentOK, _ := release(e, service, cluster, maxDeployChecks, *newTD.TaskDefinitionArn) + + if !deploymentOK { + log.Println("Rolling back failed deployment") + rollbackOK, _ := release(e, service, cluster, maxDeployChecks, *currTD.TaskDefinitionArn) + + if !rollbackOK { + log.Println("Error rolling back") + } + return errors.New("deploy failed") + } + + log.Println("Deployment succeeded") + return nil +} diff --git a/pkg/deploy/scaling.go b/pkg/deploy/scaling.go new file mode 100644 index 0000000..d934fc5 --- /dev/null +++ b/pkg/deploy/scaling.go @@ -0,0 +1,73 @@ +package deploy + +import ( + "context" + "fmt" + + "github.com/assemblyai/drone-deploy-ecs/pkg/types" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" + astypes "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" +) + +// TODO add logging +func AppAutoscalingTargetExists() bool { + + return true +} + +func GetServiceMaxCount(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (int32, error) { + r, err := getScalableTarget(ctx, c, cluster, service) + + if err != nil { + return 0, err + } + + return *r.MaxCapacity, nil +} + +func ScaleDown(c types.ECSClient, desiredCount int32, minCount int32, maxCount int32) { + // Set max, desired, min to 0 +} + +// +func ScaleUp(c types.ECSClient, count int32, maxCount int32) { + // Set max count to maxCount +} + +func setAppAutoscalingMaxCount(ctx context.Context, c types.AppAutoscalingClient, service string, cluster string, maxCount int32) error { + + p := applicationautoscaling.RegisterScalableTargetInput{ + ResourceId: aws.String(fmt.Sprintf("service/%s/%s", cluster, service)), + MaxCapacity: &maxCount, + } + + _, err := c.RegisterScalableTarget(ctx, &p) + + if err != nil { + return err + } + + return nil +} + +func setserviceDesiredCount() { + +} + +func getScalableTarget(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (*astypes.ScalableTarget, error) { + resourceID := fmt.Sprintf("service/%s/%s", cluster, service) + + p := applicationautoscaling.DescribeScalableTargetsInput{ + ServiceNamespace: "ecs", + ResourceIds: []string{resourceID}, + } + + r, err := c.DescribeScalableTargets(ctx, &p, nil) + + if err != nil { + return nil, err + } + + return &r.ScalableTargets[0], nil +} diff --git a/pkg/deploy/service.go b/pkg/deploy/service.go index 47ca00c..0ad56dc 100644 --- a/pkg/deploy/service.go +++ b/pkg/deploy/service.go @@ -3,7 +3,6 @@ package deploy import ( "context" "errors" - "fmt" "log" "github.com/assemblyai/drone-deploy-ecs/pkg/types" @@ -49,12 +48,6 @@ func GetServiceDesiredCount(ctx context.Context, c types.ECSClient, service stri return out.Services[0].DesiredCount, nil } -func GetServiceMaxCount(c types.AppAutoscalingClient, cluster string, service string) { - resourceID := fmt.Sprintf("service/%s/%s", cluster, service) - // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/applicationautoscaling#Client.DescribeScalableTargets - -} - func UpdateServiceTaskDefinitionVersion(ctx context.Context, c types.ECSClient, service string, cluster string, taskDefinitonARN string) (string, error) { i := ecs.UpdateServiceInput{ @@ -128,15 +121,3 @@ func GreenScaleUpFinished(ctx context.Context, c types.ECSClient, service string return true, nil } - -func ScaleDown(c types.ECSClient, desiredCount int32, minCount int32, maxCount int32) { - // Set max, desired, min to 0 -} - -func ScaleUp(c types.ECSClient, count int32, maxCount int32) { - // Set max count to maxCount -} - -func GetContainerStatus() { - -} diff --git a/pkg/types/types.go b/pkg/types/types.go index 854129b..0ae71b8 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -3,6 +3,7 @@ package types import ( "context" + "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/ecs" ) @@ -14,4 +15,6 @@ type ECSClient interface { } type AppAutoscalingClient interface { + DescribeScalableTargets(ctx context.Context, params *applicationautoscaling.DescribeScalableTargetsInput, optFns ...func(*applicationautoscaling.Options)) (*applicationautoscaling.DescribeScalableTargetsOutput, error) + RegisterScalableTarget(ctx context.Context, params *applicationautoscaling.RegisterScalableTargetInput, optFns ...func(*applicationautoscaling.Options)) (*applicationautoscaling.RegisterScalableTargetOutput, error) } From 14171796693641f403789749bf03b424d554ccfd Mon Sep 17 00:00:00 2001 From: mitchya1 Date: Fri, 11 Jun 2021 15:42:27 -0600 Subject: [PATCH 3/5] cleanup, bug fixes, add tests, update docs --- .env.example | 10 +- README.md | 71 +++++++- cmd/plugin/blue_green.go | 269 +++++++++++++++++++++++----- cmd/plugin/blue_green_test.go | 49 +++++ cmd/plugin/helpers.go | 10 +- cmd/plugin/main.go | 23 ++- go.mod | 3 +- pkg/deploy/errors.go | 9 + pkg/deploy/mock_app_autoscaling.go | 58 ++++++ pkg/deploy/{mock.go => mock_ecs.go} | 3 +- pkg/deploy/scaling.go | 79 +++++--- pkg/deploy/scaling_test.go | 173 ++++++++++++++++++ pkg/deploy/service.go | 21 ++- pkg/deploy/service_test.go | 79 ++++++++ pkg/deploy/structs.go | 12 ++ 15 files changed, 780 insertions(+), 89 deletions(-) create mode 100644 cmd/plugin/blue_green_test.go create mode 100644 pkg/deploy/errors.go create mode 100644 pkg/deploy/mock_app_autoscaling.go rename pkg/deploy/{mock.go => mock_ecs.go} (98%) create mode 100644 pkg/deploy/scaling_test.go create mode 100644 pkg/deploy/structs.go diff --git a/.env.example b/.env.example index 650f8b6..1dd2980 100644 --- a/.env.example +++ b/.env.example @@ -2,4 +2,12 @@ export PLUGIN_AWS_REGION=us-east-2 export PLUGIN_SERVICE= export PLUGIN_CLUSTER= export PLUGIN_CONTAINER= -export PLUGIN_IMAGE= \ No newline at end of file +export PLUGIN_IMAGE= +export PLUGIN_MODE= +export PLUGIN_BLUE_SERVICE= +export PLUGIN_GREEN_SERVICE= +export PLUGIN_MAX_DEPLOY_CHECKS= +export PLUGIN_SCALE_DOWN_PERCENT= +export PLUGIN_SCALE_DOWN_INTERVAL= +export PLUGIN_SCALE_DOWN_WAIT_PERIOD= +export PLUGIN_CHECKS_TO_PASS= \ No newline at end of file diff --git a/README.md b/README.md index 7b91504..84b8537 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,20 @@ `drone-deploy-ecs` is an opinionated Drone plugin for updating a single container within an ECS Task. -During deployment, the plugin retrieves the active Task Definition for a specified ECS Service, creates a new revision of the Task Definition with an updated image for a specified container, updates the Service to use the new Task Definition, and waits for the deployment to complete. +This plugin has support for two deployment modes: rolling and blue / green. + +During a rolling deployment, the plugin retrieves the active Task Definition for a specified ECS Service, creates a new revision of the Task Definition with an updated image for a specified container, updates the Service to use the new Task Definition, and waits for the deployment to complete. + +A blue / green deployment is similar to a rolling deployment. The key difference is that once the number of running green tasks matches the number of desired green tasks, the blue service is scaled down. It's important to note that this plugin _only_ uses desired vs running to determine deployment health. It will not check the health of a target in a target group, for example [ECR Link](https://gallery.ecr.aws/assemblyai/drone-deploy-ecs) ## Important Notes -This plugin cannot update different containers within the same Task Definition simultaneously. It will only update the image for a single container within a Task Defintion +This plugin cannot update multiple containers within the same Task Definition simultaneously. It will only update the image for a single container within a Task Defintion The ECS Service must use the `ECS` deployment controller. -~~This plugin will not rollback for you. For rollbacks, use a [deployment circuit breaker](https://aws.amazon.com/blogs/containers/announcing-amazon-ecs-deployment-circuit-breaker/).~~ ## Requirements @@ -28,10 +31,13 @@ The ECS Service being deployed to must use the `ECS` deployment controller. - `ecs:DescribeServices` on any services this tool will modify - `ecs:UpdateService` on any services this tool will modify - `ecs:RegisterTaskDefinition` on `*` - +- `application-autoscaling:DescribeScalableTargets` on `*` +- `application-autoscaling:RegisterScalableTarget` on `*` if you plan on using a blue/green deployment ## Example usage +### Rolling Deployment + ```yaml --- kind: pipeline @@ -41,7 +47,6 @@ steps: - name: deploy image: public.ecr.aws/assemblyai/drone-deploy-ecs settings: - # Can either be rolling or blue-green mode: rolling aws_region: us-east-2 # The name of the ECS service @@ -55,8 +60,58 @@ steps: max_deploy_checks: 10 ``` -## Blue / Green +### Blue / Green + +Blue / Green deployments will work with services that use Application Autoscaling and those that do not. + +One service must have a desired count of 0, the other must have a desired count > 0. + +It does not matter which service is set for `blue_service` or `green_service`. The plugin will use the service with a desired count of 0 as the green service. This is simply a way to define which two services the plugin should modify. + +Once the number of running containers equals the number of desired containers for the green service, the plugin will begin scaling down the blue service by `scale_down_percent`. + + +```yml +--- +kind: pipeline +name: deploy + +steps: +- name: deploy + image: public.ecr.aws/assemblyai/drone-deploy-ecs + settings: + mode: blue-green + aws_region: us-east-2 + # The name of the green ECS service + green_service: webapp-green + # The name of the blue ECS service + blue_service: webapp-blue + # The name of the ECS cluster that the service is in + cluster: dev-ecs-cluster + # The name of the container to update + container: nginx + # The image to deploy + image: myorg/nginx-${DRONE_COMMIT_SHA} + # How many times to check rollout status before failing + max_deploy_checks: 10 + # Percent of instances to scale down blue service by + scale_down_percent: 50 + # Seconds to wait between scale down events + scale_down_interval: 600 + # Number of seconds between scaling up green service and scaling down blue + # This is useful if your application takes some time to become healthy + scale_down_wait_period: 10 + # Number of times running count must equal desired count in order to mark green deployment as a success + checks_to_pass: 2 +``` + -The ECS service must have an associated Application Autoscaling Target +## TODO -The deployment type must be ECS +- Fix bug setting max count to 0 too early when app autoscaling is in use +- Code cleanup +- Better `settings` documentation +- Tests +- Better, more consistent logging +- Update `pkg/deploy` functions to use `deploy.DeployConfig` +- Fix `scaleDownInPercentages()` bug \ No newline at end of file diff --git a/cmd/plugin/blue_green.go b/cmd/plugin/blue_green.go index a37b739..71e5bdd 100644 --- a/cmd/plugin/blue_green.go +++ b/cmd/plugin/blue_green.go @@ -5,39 +5,16 @@ import ( "errors" "log" "os" + "strconv" "time" "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" "github.com/assemblyai/drone-deploy-ecs/pkg/types" ) -/* -Blue / Green - -When we do a blue/green deployment, we need to discover which service is "green". As far as the plugin is concerned, the service with 0 replicas is green - -If we can't decide which service is blue and which is green, we should exit with a reconcile error - -After we decide which service to update first, we need to modify the task definition. This is the same as a rolling deployment. - -Now that we know which service is blue, we need to figure out how many replicas it has. We'll set green to have the same - -After the task definition is updated, we need to set the green service to use the new TD version - -Once green is using new task definition, we need to check blue for an app autoscaling target. If it exists, -we need to set green to have the same max. Otherwise we'll just work off desired count - -Once we've done that, set green's desired count to blue's and watch until green's running count == desired count. This step will need to -have a timeout on it. If we reach the timeout, scale green back down and fail the deployment - -Once green is up and running, scale down blue. If there's an autoscaling target, scale down by decrementing the max, otherwise just -work off of the desired count - - -Once the green service is healthy, we can scale down blue -*/ - +// Returns blue service, green service, error func determineBlueGreen(e types.ECSClient, blueService string, greenService string, cluster string) (string, string, error) { + blueCount, err := deploy.GetServiceDesiredCount(context.Background(), e, blueService, cluster) if err != nil { @@ -52,44 +29,51 @@ func determineBlueGreen(e types.ECSClient, blueService string, greenService stri return "", "", errors.New("deploy failed") } + log.Printf("Service '%s' has a desired count of '%d'\n", blueService, blueCount) + log.Printf("Service '%s' has a desired count of '%d'\n", greenService, greenCount) + if blueCount == 0 { - return blueService, greenService, nil + return greenService, blueService, nil } if greenCount == 0 { - return greenService, blueService, nil + return blueService, greenService, nil } log.Println("Unable to determine which service is blue and which is green") - log.Printf("Service '%s' has %d desired replicas while service '%s' has %d desired replicas. One of these should be 0", blueService, blueCount, greenService, greenCount) + log.Printf("Service '%s' has %d desired replicas while service '%s' has %d desired replicas. One of these should be 0\n", blueService, blueCount, greenService, greenCount) return "", "", errors.New("reconcile error") } -func blueGreen(e types.ECSClient, cluster string, container string, image string) error { - blueServiceName := os.Getenv("PLUGIN_BLUE_SERVICE_NAME") - greenServiceName := os.Getenv("PLUGIN_GREEN_SERVICE_NAME") +func blueGreen(dc deploy.DeployConfig, maxDeployChecks int) error { + log.Println("Beginning blue green deployment") + + blueServiceName := os.Getenv("PLUGIN_BLUE_SERVICE") + greenServiceName := os.Getenv("PLUGIN_GREEN_SERVICE") - determinedBlueService, determinedGreenService, err := determineBlueGreen(e, blueServiceName, greenServiceName, cluster) + determinedBlueService, determinedGreenService, err := determineBlueGreen(dc.ECS, blueServiceName, greenServiceName, dc.Cluster) if err != nil { return err } - td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), e, determinedBlueService, cluster) + log.Printf("Determined service '%s' is blue and '%s' is green\n", determinedBlueService, determinedGreenService) + + td, err := deploy.GetServiceRunningTaskDefinition(context.TODO(), dc.ECS, determinedBlueService, dc.Cluster) if err != nil { log.Println("Failing because of an error determining the currently in-use task definition") return err } - currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), e, td) + currTD, err := deploy.RetrieveTaskDefinition(context.TODO(), dc.ECS, td) if err != nil { log.Println("Failing because of an error retrieving the currently in-use task definition") return err } - newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), e, currTD, container, image) + newTD, err := deploy.CreateNewTaskDefinitionRevision(context.TODO(), dc.ECS, currTD, dc.Container, dc.Image) if err != nil { log.Println("Failing because of an error retrieving the creating a new task definition revision") @@ -98,30 +82,225 @@ func blueGreen(e types.ECSClient, cluster string, container string, image string log.Println("Created new task definition revision", newTD.Revision) - currBlueDesiredCount, err := deploy.GetServiceDesiredCount(context.Background(), e, determinedBlueService, cluster) + currBlueDesiredCount, err := deploy.GetServiceDesiredCount(context.Background(), dc.ECS, determinedBlueService, dc.Cluster) if err != nil { log.Println("Failing because of an error determining desired count for blue service", err.Error()) return err } - // There is no deployment so discard it - _, err = deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), e, determinedGreenService, cluster, *newTD.TaskDefinitionArn) + // There is no deployment ID so discard it + _, err = deploy.UpdateServiceTaskDefinitionVersion(context.TODO(), dc.ECS, determinedGreenService, dc.Cluster, *newTD.TaskDefinitionArn) if err != nil { log.Println("Error updating task definition for service", err.Error()) return errors.New("deploy failed") } + serviceUsesAppAutoscaling, err := deploy.AppAutoscalingTargetExists(context.Background(), dc.AppAutoscaling, dc.Cluster, determinedBlueService) + + if err != nil { + log.Println("Error determining if service uses application autoscaling", err.Error()) + return err + } + + var serviceMaxCount int32 + var serviceMinCount int32 + + if serviceUsesAppAutoscaling { + log.Printf("Service '%s' uses application autoscaling. Will modify autoscaling max count", determinedGreenService) + serviceMaxCount, serviceMinCount, err = deploy.GetServiceMinMaxCount(context.Background(), dc.AppAutoscaling, dc.Cluster, determinedBlueService) + + if err != nil { + log.Println("Error determining service max count", err.Error()) + return err + } + } else { + serviceMaxCount = -1 + serviceMinCount = 0 + } + // Scale up green service to the same count as blue - // TODO pass the actual max instead of 0 - deploy.ScaleUp(e, currBlueDesiredCount, 0) - log.Println("Pausing for 10 seconds while ECS schedules", currBlueDesiredCount, " containers") - time.Sleep(10 * time.Second) + dc.ScaleUp(currBlueDesiredCount, serviceMinCount, serviceMaxCount, determinedGreenService) + + log.Println("Pausing for 45 seconds while ECS schedules", currBlueDesiredCount, "containers") + time.Sleep(45 * time.Second) + + // Start polling deployment + greenScaleupFinished := false + deployCounter := 0 + successCounter := 0 + + successCountThreshold, _ := strconv.Atoi(os.Getenv("PLUGIN_CHECKS_TO_PASS")) + + greenScaleupFinished, err = dc.GreenScaleUpFinished(context.Background(), determinedGreenService) + + if err != nil { + log.Println("Error checking if green service has finished scaling", err.Error()) + return err + } + + for { + // Ensure we haven't surpassed the max check limit + if deployCounter > maxDeployChecks { + log.Println("Max deploy checks surpassed. Scaling green down and marking deployment a failure") + dc.ScaleDown(0, 0, 0, determinedGreenService, serviceUsesAppAutoscaling) + return errors.New("deploy failed") + } + + if !greenScaleupFinished { + // In this case, the service is not done scaling up + // Increment counter first + deployCounter++ + + // Check if scale up has finished + greenScaleupFinished, err = dc.GreenScaleUpFinished(context.Background(), determinedGreenService) + + if err != nil { + log.Println("Error checking if green has finished scaling up", err.Error()) + return err + } + // Wait for 10 seconds + log.Println("Waiting 10 seconds for green service to scale up") + time.Sleep(10 * time.Second) + // Reset successCounter, successful checks must be consecutive + successCounter = 0 + + } else { + // In this case, running == desired + // Now we need to make sure the healthy check threshold has been reached + if successCounter < successCountThreshold { + // We simply need to increment the counter here + // because we already know that running == desired + log.Println("Successful checks:", successCounter) + // Wait for 10 seconds + log.Println("Waiting 10 seconds before incrementing healthy checks") + time.Sleep(10 * time.Second) + successCounter++ + } else { + // Again, running == desired + // _and_ successCounter >= successCountThreshold + log.Println("Green deployment has reached healthy check threshold") + break + } + } + + } + + log.Printf("Green service '%s' finished scaling up! Scaling down blue service '%s'\n", determinedGreenService, determinedBlueService) + + log.Printf("Waiting %s seconds before scaling down blue", os.Getenv("PLUGIN_SCALE_DOWN_WAIT_PERIOD")) + + scaleDownPause, _ := strconv.Atoi(os.Getenv("PLUGIN_SCALE_DOWN_WAIT_PERIOD")) - // Loop and check GreenScaleUpFinished - timeout and fail after maxDeployChecks + time.Sleep(time.Duration(scaleDownPause) * time.Second) - // Scale down + err = scaleDownInPercentages( + dc, + determinedBlueService, + serviceUsesAppAutoscaling, + os.Getenv("PLUGIN_SCALE_DOWN_PERCENT"), + os.Getenv("PLUGIN_SCALE_DOWN_INTERVAL"), + int(currBlueDesiredCount), + ) + + return err +} + +/* +There is a bug in this function. It effectively performs an exponential backoff. +This is because it multiplies desiredCount * scalePercent, but it also calls itself recursively, +so desiredCount gets smaller and smaller + +We should hold the initial desiredCount passed from blueGreen(), maybe in a global variable, then perform our logic on that number + +This bug doesn't really hurt anything, it just makes the scale down go slower than expected +*/ +func scaleDownInPercentages(dc deploy.DeployConfig, service string, serviceUsesAppAutoscaling bool, scalePercent string, scaleDownInterval string, desiredCount int) error { + scalePercentString, err := strconv.Atoi(scalePercent) + + if err != nil { + log.Println("Error converting scale down percent", scalePercent, "to integer. Failing.") + return err + } + + scaleDownWait, err := strconv.Atoi(scaleDownInterval) + + if err != nil { + log.Println("Error converting scale down interval", scaleDownInterval, "to integer. Failing.") + return err + } + + scalePercentAsFloat := float64(scalePercentString) + + // Convert int to decimal + percent := float64(scalePercentAsFloat) / float64(100) + + log.Printf("Scaling down by %d percent\n", int(percent*100)) + + var scaleDownBy int + var newDesiredCount int32 + var lastScaleDownEvent bool + + scaleDownNumber := float64(desiredCount) * percent + + if scaleDownNumber < 0 { + scaleDownBy = 1 + } else if scaleDownNumber > 0 && scaleDownNumber < 1 { + // Handle a bug where, if running count is less than 10 + // the desired count would be set to 0 + log.Println("Number of containers to remove is a decimal between 0 and 1. Removing one container") + scaleDownBy = 1 + } else { + // int() will give us a round number + scaleDownBy = int(scaleDownNumber) + } + + calculatedDesiredCount := desiredCount - scaleDownBy + + if calculatedDesiredCount <= 0 { + newDesiredCount = 0 + lastScaleDownEvent = true + } else { + newDesiredCount = int32(calculatedDesiredCount) + lastScaleDownEvent = false + } + + err = dc.ScaleDown(newDesiredCount, 0, newDesiredCount, service, serviceUsesAppAutoscaling) + + if err != nil { + log.Println("Error scaling down service", err.Error()) + return err + } + + status, err := dc.GreenScaleUpFinished(context.Background(), service) + + if err != nil { + log.Println("Error checking scale down status", err.Error()) + return err + } + + for !status { + status, err = dc.GreenScaleUpFinished(context.Background(), service) + + if err != nil { + log.Println("Error checking scale down status", err.Error()) + return err + } + time.Sleep(15 * time.Second) + } + + log.Println("Finished scaling blue service down to", newDesiredCount) + + if lastScaleDownEvent { + log.Println("Scale down complete") + return nil + } else { + log.Println("Waiting", scaleDownWait, "seconds before scaling down again") + time.Sleep(time.Duration(scaleDownWait) * time.Second) + + scaleDownInPercentages(dc, service, serviceUsesAppAutoscaling, scalePercent, scaleDownInterval, int(newDesiredCount)) + } return nil } diff --git a/cmd/plugin/blue_green_test.go b/cmd/plugin/blue_green_test.go new file mode 100644 index 0000000..66bf538 --- /dev/null +++ b/cmd/plugin/blue_green_test.go @@ -0,0 +1,49 @@ +package main + +// Need to refactor the mock functions to allow this to work +/* +func Test_determineBlueGreen(t *testing.T) { + type args struct { + e types.ECSClient + blueService string + greenService string + cluster string + } + tests := []struct { + name string + args args + want string + want1 string + wantErr bool + }{ + { + name: "success", + args: args{ + e: deploy.MockECSClient{}, + blueService: "test-cluster", + greenService: "test-cluster-green", + cluster: "test-cluster", + }, + want: "test-cluster", + want1: "test-cluster-green", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := determineBlueGreen(tt.args.e, tt.args.blueService, tt.args.greenService, tt.args.cluster) + if (err != nil) != tt.wantErr { + t.Errorf("determineBlueGreen() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("determineBlueGreen() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("determineBlueGreen() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +*/ diff --git a/cmd/plugin/helpers.go b/cmd/plugin/helpers.go index a7a9193..79ba526 100644 --- a/cmd/plugin/helpers.go +++ b/cmd/plugin/helpers.go @@ -48,8 +48,12 @@ func parseRollingVars() error { func checkBlueGreenVars() error { requiredVars := []string{ - "PLUGIN_BLUE_SERVICE_NAME", - "PLUGIN_GREEN_SERVICE_NAME", + "PLUGIN_BLUE_SERVICE", + "PLUGIN_GREEN_SERVICE", + "PLUGIN_SCALE_DOWN_PERCENT", + "PLUGIN_SCALE_DOWN_INTERVAL", + "PLUGIN_SCALE_DOWN_WAIT_PERIOD", + "PLUGIN_CHECKS_TO_PASS", } for _, v := range requiredVars { @@ -75,7 +79,7 @@ func newECSClient(region string) *ecs.Client { return ecs.NewFromConfig(cfg) } -func newAppAutoscalingTarget(region string) *applicationautoscaling.Client { +func newAppAutoscalingClient(region string) *applicationautoscaling.Client { cfg, err := config.LoadDefaultConfig( context.TODO(), config.WithRegion(region), diff --git a/cmd/plugin/main.go b/cmd/plugin/main.go index 2883de4..efd37f9 100644 --- a/cmd/plugin/main.go +++ b/cmd/plugin/main.go @@ -4,6 +4,8 @@ import ( "log" "os" "strconv" + + "github.com/assemblyai/drone-deploy-ecs/pkg/deploy" ) const ( @@ -16,8 +18,6 @@ func main() { os.Exit(1) } - e := newECSClient(os.Getenv("PLUGIN_AWS_REGION")) - var maxDeployChecks int if os.Getenv("PLUGIN_MAX_DEPLOY_CHECKS") == "" { @@ -33,14 +33,23 @@ func main() { } } - cluster := os.Getenv("PLUGIN_CLUSTER") - container := os.Getenv("PLUGIN_CONTAINER") - image := os.Getenv("PLUGIN_IMAGE") + dc := deploy.DeployConfig{ + ECS: newECSClient(os.Getenv("PLUGIN_AWS_REGION")), + AppAutoscaling: newAppAutoscalingClient(os.Getenv("PLUGIN_AWS_REGION")), + Cluster: os.Getenv("PLUGIN_CLUSTER"), + Container: os.Getenv("PLUGIN_CONTAINER"), + Image: os.Getenv("PLUGIN_IMAGE"), + } if os.Getenv("PLUGIN_MODE") == "blue-green" { - + if err := checkBlueGreenVars(); err != nil { + os.Exit(1) + } + if err := blueGreen(dc, maxDeployChecks); err != nil { + os.Exit(1) + } } else { - if err := rolling(e, cluster, container, image, maxDeployChecks); err != nil { + if err := rolling(dc.ECS, dc.Cluster, dc.Container, dc.Image, maxDeployChecks); err != nil { os.Exit(1) } } diff --git a/go.mod b/go.mod index eadc1fc..213a956 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,9 @@ go 1.16 require ( github.com/aws/aws-sdk-go-v2 v1.6.0 github.com/aws/aws-sdk-go-v2/config v1.3.0 - github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.3.1 github.com/aws/aws-sdk-go-v2/service/ecs v1.5.0 + github.com/aws/smithy-go v1.4.0 github.com/pkg/errors v0.9.1 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/pkg/deploy/errors.go b/pkg/deploy/errors.go new file mode 100644 index 0000000..c663496 --- /dev/null +++ b/pkg/deploy/errors.go @@ -0,0 +1,9 @@ +package deploy + +type ErrNoResults struct { + Message string +} + +func (e *ErrNoResults) Error() string { + return e.Message +} diff --git a/pkg/deploy/mock_app_autoscaling.go b/pkg/deploy/mock_app_autoscaling.go new file mode 100644 index 0000000..25778cb --- /dev/null +++ b/pkg/deploy/mock_app_autoscaling.go @@ -0,0 +1,58 @@ +package deploy + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" + astypes "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" + "github.com/aws/smithy-go/middleware" +) + +type MockAppAutoscalingClient struct { + TestingT *testing.T + WantError bool + TargetExists bool +} + +func (c MockAppAutoscalingClient) DescribeScalableTargets(ctx context.Context, params *applicationautoscaling.DescribeScalableTargetsInput, optFns ...func(*applicationautoscaling.Options)) (*applicationautoscaling.DescribeScalableTargetsOutput, error) { + if c.WantError { + return nil, errors.New("error") + } + + if !c.TargetExists { + return &applicationautoscaling.DescribeScalableTargetsOutput{ScalableTargets: []astypes.ScalableTarget{}}, nil + } + + out := applicationautoscaling.DescribeScalableTargetsOutput{ + NextToken: new(string), + ScalableTargets: []astypes.ScalableTarget{ + { + MaxCapacity: aws.Int32(20), + MinCapacity: aws.Int32(4), + ResourceId: new(string), + RoleARN: new(string), + ScalableDimension: astypes.ScalableDimensionECSServiceDesiredCount, + ServiceNamespace: astypes.ServiceNamespaceEcs, + SuspendedState: &astypes.SuspendedState{}, + }, + }, + ResultMetadata: middleware.Metadata{}, + } + + return &out, nil +} + +func (c MockAppAutoscalingClient) RegisterScalableTarget(ctx context.Context, params *applicationautoscaling.RegisterScalableTargetInput, optFns ...func(*applicationautoscaling.Options)) (*applicationautoscaling.RegisterScalableTargetOutput, error) { + if c.WantError { + return nil, errors.New("error") + } + + out := applicationautoscaling.RegisterScalableTargetOutput{ + ResultMetadata: middleware.Metadata{}, + } + + return &out, nil +} diff --git a/pkg/deploy/mock.go b/pkg/deploy/mock_ecs.go similarity index 98% rename from pkg/deploy/mock.go rename to pkg/deploy/mock_ecs.go index 64d3be6..a8328db 100644 --- a/pkg/deploy/mock.go +++ b/pkg/deploy/mock_ecs.go @@ -91,10 +91,11 @@ func (c MockECSClient) DescribeServices(ctx context.Context, params *ecs.Describ s := []ecstypes.Service{ { - ServiceName: aws.String("ci-cluster"), + ServiceName: aws.String("test-cluster"), Status: aws.String("ACTIVE"), Deployments: d, TaskDefinition: aws.String(testTDARN), + DesiredCount: 2, }, } diff --git a/pkg/deploy/scaling.go b/pkg/deploy/scaling.go index d934fc5..b474388 100644 --- a/pkg/deploy/scaling.go +++ b/pkg/deploy/scaling.go @@ -3,6 +3,7 @@ package deploy import ( "context" "fmt" + "log" "github.com/assemblyai/drone-deploy-ecs/pkg/types" "github.com/aws/aws-sdk-go-v2/aws" @@ -10,36 +11,74 @@ import ( astypes "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" ) -// TODO add logging -func AppAutoscalingTargetExists() bool { +func AppAutoscalingTargetExists(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (bool, error) { + _, err := getScalableTarget(ctx, c, cluster, service) + + if err == nil { + return true, nil + } + + if _, ok := err.(*ErrNoResults); ok { + return false, nil + } else { + return false, err + } - return true } -func GetServiceMaxCount(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (int32, error) { +func GetServiceMinMaxCount(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (int32, int32, error) { r, err := getScalableTarget(ctx, c, cluster, service) if err != nil { - return 0, err + return 0, 0, err } - return *r.MaxCapacity, nil + return *r.MaxCapacity, *r.MinCapacity, nil } -func ScaleDown(c types.ECSClient, desiredCount int32, minCount int32, maxCount int32) { - // Set max, desired, min to 0 +func (c DeployConfig) ScaleDown(desiredCount int32, minCount int32, maxCount int32, service string, serviceUsesAppAutoscaling bool) error { + log.Println("Setting desired count to", desiredCount, "for service", service) + err := setECSServiceDesiredCount(c.ECS, service, c.Cluster, desiredCount) + + if err != nil { + return err + } + + if serviceUsesAppAutoscaling { + log.Println("Setting max count to", maxCount, "for service", service) + return setAppAutoscalingCounts(context.Background(), c.AppAutoscaling, service, c.Cluster, maxCount, 0) + } + + return nil } -// -func ScaleUp(c types.ECSClient, count int32, maxCount int32) { - // Set max count to maxCount +func (c DeployConfig) ScaleUp(desiredCount int32, minCount int32, maxCount int32, service string) error { + if maxCount == -1 { + log.Println("Setting desired count to", desiredCount, "for service", service) + return setECSServiceDesiredCount(c.ECS, service, c.Cluster, desiredCount) + } else { + err := setAppAutoscalingCounts(context.Background(), c.AppAutoscaling, service, c.Cluster, maxCount, minCount) + + if err != nil { + return err + } + + err = setECSServiceDesiredCount(c.ECS, service, c.Cluster, desiredCount) + + return err + } + } -func setAppAutoscalingMaxCount(ctx context.Context, c types.AppAutoscalingClient, service string, cluster string, maxCount int32) error { +func setAppAutoscalingCounts( + ctx context.Context, c types.AppAutoscalingClient, service string, cluster string, maxCount int32, minCount int32) error { p := applicationautoscaling.RegisterScalableTargetInput{ - ResourceId: aws.String(fmt.Sprintf("service/%s/%s", cluster, service)), - MaxCapacity: &maxCount, + ResourceId: aws.String(fmt.Sprintf("service/%s/%s", cluster, service)), + ServiceNamespace: astypes.ServiceNamespaceEcs, + MaxCapacity: &maxCount, + MinCapacity: &minCount, + ScalableDimension: astypes.ScalableDimensionECSServiceDesiredCount, } _, err := c.RegisterScalableTarget(ctx, &p) @@ -51,23 +90,23 @@ func setAppAutoscalingMaxCount(ctx context.Context, c types.AppAutoscalingClient return nil } -func setserviceDesiredCount() { - -} - func getScalableTarget(ctx context.Context, c types.AppAutoscalingClient, cluster string, service string) (*astypes.ScalableTarget, error) { resourceID := fmt.Sprintf("service/%s/%s", cluster, service) p := applicationautoscaling.DescribeScalableTargetsInput{ - ServiceNamespace: "ecs", + ServiceNamespace: astypes.ServiceNamespaceEcs, ResourceIds: []string{resourceID}, } - r, err := c.DescribeScalableTargets(ctx, &p, nil) + r, err := c.DescribeScalableTargets(ctx, &p) if err != nil { return nil, err } + if len(r.ScalableTargets) == 0 { + return nil, &ErrNoResults{Message: "No scalable targets"} + } + return &r.ScalableTargets[0], nil } diff --git a/pkg/deploy/scaling_test.go b/pkg/deploy/scaling_test.go new file mode 100644 index 0000000..d98b72b --- /dev/null +++ b/pkg/deploy/scaling_test.go @@ -0,0 +1,173 @@ +package deploy + +import ( + "context" + "testing" + + "github.com/assemblyai/drone-deploy-ecs/pkg/types" +) + +func TestAppAutoscalingTargetExists(t *testing.T) { + type args struct { + ctx context.Context + c types.AppAutoscalingClient + cluster string + service string + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "target-does-not-exist", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: false, TestingT: t, TargetExists: false}, + cluster: "test-cluster", + service: "ci-service-green", + }, + want: false, + wantErr: false, + }, + { + name: "target-exists", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: false, TestingT: t, TargetExists: true}, + cluster: "test-cluster", + service: "ci-service-green", + }, + want: true, + wantErr: false, + }, + { + name: "error", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: true, TestingT: t, TargetExists: true}, + cluster: "test-cluster", + service: "ci-service-green", + }, + want: false, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := AppAutoscalingTargetExists(tt.args.ctx, tt.args.c, tt.args.cluster, tt.args.service) + if (err != nil) != tt.wantErr { + t.Errorf("AppAutoscalingTargetExists() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("AppAutoscalingTargetExists() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetServiceMinMaxCount(t *testing.T) { + type args struct { + ctx context.Context + c types.AppAutoscalingClient + cluster string + service string + } + tests := []struct { + name string + args args + want int32 + want1 int32 + wantErr bool + }{ + { + name: "error", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: true, TestingT: t, TargetExists: false}, + cluster: "test-cluster", + service: "ci-service-green", + }, + want: 0, + want1: 0, + wantErr: true, + }, + { + name: "correct-count", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: false, TestingT: t, TargetExists: true}, + cluster: "test-cluster", + service: "ci-service-green", + }, + want: 20, + want1: 4, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := GetServiceMinMaxCount(tt.args.ctx, tt.args.c, tt.args.cluster, tt.args.service) + if (err != nil) != tt.wantErr { + t.Errorf("GetServiceMinMaxCount() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetServiceMinMaxCount() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("GetServiceMinMaxCount() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_setAppAutoscalingCounts(t *testing.T) { + type args struct { + ctx context.Context + c types.AppAutoscalingClient + service string + cluster string + maxCount int32 + minCount int32 + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "success", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: false, TestingT: t}, + cluster: "test-cluster", + service: "ci-service-green", + maxCount: 50, + minCount: 10, + }, + wantErr: false, + }, + { + name: "error", + args: args{ + ctx: context.Background(), + c: MockAppAutoscalingClient{WantError: true, TestingT: t}, + cluster: "test-cluster", + service: "ci-service-green", + maxCount: 50, + minCount: 10, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := setAppAutoscalingCounts(tt.args.ctx, tt.args.c, tt.args.service, tt.args.cluster, tt.args.maxCount, tt.args.minCount); (err != nil) != tt.wantErr { + t.Errorf("setAppAutoscalingCounts() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/deploy/service.go b/pkg/deploy/service.go index 0ad56dc..3fac432 100644 --- a/pkg/deploy/service.go +++ b/pkg/deploy/service.go @@ -99,13 +99,28 @@ func CheckDeploymentStatus(ctx context.Context, c types.ECSClient, service strin } -func GreenScaleUpFinished(ctx context.Context, c types.ECSClient, service string, cluster string) (bool, error) { +func setECSServiceDesiredCount(c types.ECSClient, service string, cluster string, desiredCount int32) error { + + p := ecs.UpdateServiceInput{ + Service: &service, + DesiredCount: &desiredCount, + Cluster: &cluster, + } + + // TODO use provided context + _, err := c.UpdateService(context.Background(), &p) + + return err +} + +// TODO update mock client so we can test this +func (c DeployConfig) GreenScaleUpFinished(ctx context.Context, service string) (bool, error) { i := ecs.DescribeServicesInput{ Services: []string{service}, - Cluster: aws.String(cluster), + Cluster: aws.String(c.Cluster), } - out, err := c.DescribeServices( + out, err := c.ECS.DescribeServices( ctx, &i, ) diff --git a/pkg/deploy/service_test.go b/pkg/deploy/service_test.go index 4e2af38..988ecf3 100644 --- a/pkg/deploy/service_test.go +++ b/pkg/deploy/service_test.go @@ -151,3 +151,82 @@ func TestUpdateServiceTaskDefinitionVersion(t *testing.T) { }) } } + +func Test_setECSServiceDesiredCount(t *testing.T) { + type args struct { + c types.ECSClient + service string + cluster string + desiredCount int32 + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "success", + args: args{ + c: MockECSClient{TestingT: t, WantError: false}, + service: "test-service", + cluster: "test-cluster", + }, + wantErr: false, + }, + { + name: "failure", + args: args{ + c: MockECSClient{TestingT: t, WantError: true}, + service: "test-service", + cluster: "test-cluster", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := setECSServiceDesiredCount(tt.args.c, tt.args.service, tt.args.cluster, tt.args.desiredCount); (err != nil) != tt.wantErr { + t.Errorf("setECSServiceDesiredCount() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGetServiceDesiredCount(t *testing.T) { + type args struct { + ctx context.Context + c types.ECSClient + service string + cluster string + } + tests := []struct { + name string + args args + want int32 + wantErr bool + }{ + { + name: "desired-count-is-2", + args: args{ + ctx: context.Background(), + c: MockECSClient{WantError: false, TestingT: t}, + cluster: "test-cluster", + service: "test-service", + }, + want: 2, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetServiceDesiredCount(tt.args.ctx, tt.args.c, tt.args.service, tt.args.cluster) + if (err != nil) != tt.wantErr { + t.Errorf("GetServiceDesiredCount() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetServiceDesiredCount() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/deploy/structs.go b/pkg/deploy/structs.go new file mode 100644 index 0000000..5ea3826 --- /dev/null +++ b/pkg/deploy/structs.go @@ -0,0 +1,12 @@ +package deploy + +import "github.com/assemblyai/drone-deploy-ecs/pkg/types" + +type DeployConfig struct { + ECS types.ECSClient + AppAutoscaling types.AppAutoscalingClient + Cluster string + Container string + Image string + // Logger +} From e62e42edca9579f4ceffe54d5db9afbf26ca9f9d Mon Sep 17 00:00:00 2001 From: mitchya1 Date: Fri, 11 Jun 2021 15:43:20 -0600 Subject: [PATCH 4/5] remove todo [CI SKIP] --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 84b8537..db6651b 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ steps: ## TODO -- Fix bug setting max count to 0 too early when app autoscaling is in use - Code cleanup - Better `settings` documentation - Tests From c3c395023801d4a71593bdc71dd93633189df409 Mon Sep 17 00:00:00 2001 From: mitchya1 Date: Fri, 11 Jun 2021 15:45:08 -0600 Subject: [PATCH 5/5] add env var --- cmd/plugin/main_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/plugin/main_test.go b/cmd/plugin/main_test.go index 4e31fa8..9e83836 100644 --- a/cmd/plugin/main_test.go +++ b/cmd/plugin/main_test.go @@ -19,6 +19,7 @@ func TestCheckEnvVarsAllVarsSet(t *testing.T) { os.Setenv("PLUGIN_CLUSTER", "some-cluster") os.Setenv("PLUGIN_CONTAINER", "some-container-name") os.Setenv("PLUGIN_IMAGE", "some/image:with-tag") + os.Setenv("PLUGIN_MODE", "rolling") err := checkEnvVars()