Skip to content

Commit

Permalink
Clean Old ECS Clusters (aws#343)
Browse files Browse the repository at this point in the history
  • Loading branch information
sethAmazon authored Jul 21, 2023
1 parent cb1e819 commit abc5f9b
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 7 deletions.
20 changes: 19 additions & 1 deletion .github/workflows/clean-aws-resources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,4 +133,22 @@ jobs:
aws-region: ${{ matrix.region }}

- name: Clean old dedicated host
run: go run ./tool/clean/clean_host/clean_host.go --tags=clean
run: go run ./tool/clean/clean_host/clean_host.go --tags=clean

clean-ecs-clusters:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4

- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
aws-region: us-west-2

- name: Clean old ecs cluster
run: go run ./tool/clean/clean_ecs/clean_ecs.go --tags=clean
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,12 @@ require (
github.com/Jeffail/gabs v1.4.0
github.com/Rican7/retry v0.1.1-0.20160712041035-272ad122d6e5
github.com/aws/aws-sdk-go v1.44.293
github.com/aws/aws-sdk-go-v2 v1.18.0
github.com/aws/aws-sdk-go-v2 v1.19.0
github.com/aws/aws-sdk-go-v2/config v1.18.25
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.25.7
github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0
github.com/aws/aws-sdk-go-v2/service/ecs v1.28.1
github.com/aws/aws-sdk-go-v2/service/efs v1.19.7
github.com/aws/smithy-go v1.13.5
github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579
Expand Down Expand Up @@ -178,8 +179,8 @@ require (
github.com/apache/thrift v0.16.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
Expand Down
11 changes: 8 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,9 @@ github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVj
github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k=
github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.2.0 h1:scBthy70MB3m4LCMFaBcmYCyR2XWOz6MxSfdSu/+fQo=
github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw=
github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q=
Expand All @@ -220,12 +221,14 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEu
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.3 h1:0O72494cCsazjpsGfo+LXezru6PMSp0HUB1m5UfpaRU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY=
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc=
Expand All @@ -237,6 +240,8 @@ github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.13.0 h1:NfqONXoDwWtBCnkPV
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.14.0 h1:P+eF8PKkeaiTfN/VBe5GI3uNdhwCPVYCQxchRewJcWk=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0 h1:NXi4pNJWjAaiI56P1Rl8DC9A4jMNRE00WNBsDua5WRg=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0/go.mod h1:L3ZT0N/vBsw77mOAawXmRnREpEjcHd2v5Hzf7AkIH8M=
github.com/aws/aws-sdk-go-v2/service/ecs v1.28.1 h1:PxWgrtfQvct60NjxSrFsSWG/Yg1HATRKP4IeUPiLlrE=
github.com/aws/aws-sdk-go-v2/service/ecs v1.28.1/go.mod h1:eZBCsRjzc+ZX8x3h0beHOu+uxRWRwnEHzzvDgKy9v0E=
github.com/aws/aws-sdk-go-v2/service/efs v1.19.7 h1:BmyhflgczNmmuAPFhAhMQuLc9zSHiqIY5ouS+oSwxPQ=
github.com/aws/aws-sdk-go-v2/service/efs v1.19.7/go.mod h1:ENSgtHyPiYyBcTAi26Hpr8Xp636IB18qr0D5Ho8EQWA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.8.0 h1:wS94St7YDmLhrPJw3mjJfCfHHOABS3G9c//mDZRzELU=
Expand Down
139 changes: 139 additions & 0 deletions tool/clean/clean_ecs/clean_ecs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: MIT

//go:build clean
// +build clean

package main

import (
"context"
"log"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ecs"

"github.com/aws/private-amazon-cloudwatch-agent-staging/tool/clean"
)

// Clean ecs clusters if they have been open longer than 7 day
func main() {
err := cleanCluster()
if err != nil {
log.Fatalf("errors cleaning %v", err)
}
}

func cleanCluster() error {
log.Print("Begin to clean ECS Clusters")

cxt := context.Background()
defaultConfig, err := config.LoadDefaultConfig(cxt)
if err != nil {
return err
}
ecsClient := ecs.NewFromConfig(defaultConfig)

terminateClusters(cxt, ecsClient)
return err
}

func terminateClusters(ctx context.Context, client *ecs.Client) {
// you can only filter ecs by name or arn
// not regex of tag name like ec2
// describe cluster input max is 100
ecsListClusterInput := ecs.ListClustersInput{
MaxResults: aws.Int32(100),
}
for {
clusterIds := make([]*string, 0)
expirationDateCluster := time.Now().UTC().Add(clean.KeepDurationOneWeek)
listClusterOutput, err := client.ListClusters(ctx, &ecsListClusterInput)
if err != nil || listClusterOutput.ClusterArns == nil || len(listClusterOutput.ClusterArns) == 0 {
break
}
describeClustersInput := ecs.DescribeClustersInput{Clusters: listClusterOutput.ClusterArns}
describeClustersOutput, err := client.DescribeClusters(ctx, &describeClustersInput)
if err != nil || describeClustersOutput.Clusters == nil || len(describeClustersOutput.Clusters) == 0 {
break
}
for _, cluster := range describeClustersOutput.Clusters {
if !strings.HasPrefix(*cluster.ClusterName, "cwagent-integ-test-cluster-") {
continue
}
if cluster.RunningTasksCount == 0 && cluster.PendingTasksCount == 0 {
clusterIds = append(clusterIds, cluster.ClusterArn)
continue
}
describeTaskInput := ecs.DescribeTasksInput{Cluster: cluster.ClusterArn}
describeTasks, err := client.DescribeTasks(ctx, &describeTaskInput)
if err != nil {
continue
}
addCluster := true
for _, task := range describeTasks.Tasks {
if expirationDateCluster.After(*task.StartedAt) {
log.Printf("Task %s launch-date %s", *task.TaskArn, *task.StartedAt)
} else {
addCluster = false
break
}
}
if addCluster {
clusterIds = append(clusterIds, cluster.ClusterArn)
}
}
if len(clusterIds) == 0 {
log.Printf("No clusters to terminate")
return
}

for _, clusterId := range clusterIds {
log.Printf("cluster to temrinate %s", *clusterId)
listContainerInstanceInput := ecs.ListContainerInstancesInput{Cluster: clusterId}
listContainerInstances, err := client.ListContainerInstances(ctx, &listContainerInstanceInput)
if err != nil {
log.Printf("Error %v getting container instances cluster %s", err, *clusterId)
continue
}
for _, instance := range listContainerInstances.ContainerInstanceArns {
deregisterContainerInstanceInput := ecs.DeregisterContainerInstanceInput{
ContainerInstance: aws.String(instance),
Cluster: clusterId,
Force: aws.Bool(true),
}
_, err = client.DeregisterContainerInstance(ctx, &deregisterContainerInstanceInput)
if err != nil {
log.Printf("Error %v deregister container instances cluster %s container %v", err, *clusterId, instance)
continue
}
}
serviceInput := ecs.ListServicesInput{Cluster: clusterId}
services, err := client.ListServices(ctx, &serviceInput)
if err != nil {
log.Printf("Error %v getting services cluster %s", err, *clusterId)
continue
}
for _, service := range services.ServiceArns {
deleteServiceInput := ecs.DeleteServiceInput{Cluster: clusterId, Service: aws.String(service)}
_, err := client.DeleteService(ctx, &deleteServiceInput)
if err != nil {
log.Printf("Error %v deleteing service %s cluster %s", err, serviceInput, *clusterId)
continue
}
}
terminateClusterInput := ecs.DeleteClusterInput{Cluster: clusterId}
_, err = client.DeleteCluster(ctx, &terminateClusterInput)
if err != nil {
log.Printf("Error %v terminating cluster %s", err, *clusterId)
}
}
if ecsListClusterInput.NextToken == nil {
break
}
ecsListClusterInput.NextToken = listClusterOutput.NextToken
}
}
1 change: 1 addition & 0 deletions tool/clean/clean_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package clean
import "time"

const (
KeepDurationOneWeek = KeepDurationOneDay * 7
KeepDurationOneDay = -1 * time.Hour * 24
KeepDurationSixtyDay = KeepDurationOneDay * time.Duration(60)
KeepDurationTwentySixHours = KeepDurationOneDay + time.Hour*2
Expand Down

0 comments on commit abc5f9b

Please sign in to comment.