k8s-tester: improve ELB deletion

Signed-off-by: Gyuho Lee <[email protected]>

client/services.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"go.uber.org/zap"
@@ -42,15 +43,18 @@ func DeleteService(lg *zap.Logger, c k8s_client.Interface, namespace string, svc
 	return RetryWithExponentialBackOff(RetryFunction(deleteFunc, Allow(k8s_errors.IsNotFound)))
 }
 
-// WaitForServiceIngressHostname waits for Service's Ingress Hostname to be updated.
+// WaitForServiceIngressHostname waits for Service's Ingress Hostname to be updated
+// and returns the ELB ARN.
 func WaitForServiceIngressHostname(
 	lg *zap.Logger,
 	c k8s_client.Interface,
 	namespace string,
 	svcName string,
 	stopc chan struct{},
 	waitDur time.Duration,
-	opts ...OpOption) (string, error) {
+	accountID string,
+	region string,
+	opts ...OpOption) (hostName string, elbARN string, err error) {
 	ret := Op{}
 	ret.applyOpts(opts)
 
@@ -59,12 +63,11 @@ func WaitForServiceIngressHostname(
 		zap.String("service-name", svcName),
 	)
 
-	hostName := ""
 	retryStart := time.Now()
 	for time.Since(retryStart) < waitDur {
 		select {
 		case <-stopc:
-			return "", errors.New("wait for service aborted")
+			return "", "", errors.New("wait for service aborted")
 		case <-time.After(5 * time.Second):
 		}
 
@@ -111,8 +114,107 @@ func WaitForServiceIngressHostname(
 	}
 
 	if hostName == "" {
-		return "", errors.New("failed to find LoadBalancer host name")
+		return "", "", errors.New("failed to find LoadBalancer host name")
 	}
 
-	return hostName, nil
+	// TODO: find better way to find out the NLB/ELB name
+	ss := strings.Split(hostName, ".")[0]
+	ss = strings.Replace(ss, "-", "/", -1)
+	elbARN = fmt.Sprintf(
+		"arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
+		region,
+		accountID,
+		ss,
+	)
+	lg.Info("found LoadBalancer ELB ARN", zap.String("elb-arn", elbARN))
+
+	return hostName, elbARN, nil
+}
+
+// FindServiceIngressHostname finds Service's Ingress Hostname to be updated
+// and returns the ELB ARN.
+func FindServiceIngressHostname(
+	lg *zap.Logger,
+	c k8s_client.Interface,
+	namespace string,
+	svcName string,
+	stopc chan struct{},
+	waitDur time.Duration,
+	accountID string,
+	region string,
+	opts ...OpOption) (hostName string, elbARN string, exists bool, err error) {
+	ret := Op{}
+	ret.applyOpts(opts)
+
+	lg.Info("finding service ingress host name",
+		zap.String("namespace", namespace),
+		zap.String("service-name", svcName),
+	)
+
+	retryStart := time.Now()
+	for time.Since(retryStart) < waitDur {
+		select {
+		case <-stopc:
+			return "", "", true, errors.New("wait for service aborted")
+		case <-time.After(5 * time.Second):
+		}
+
+		if ret.queryFunc != nil {
+			ret.queryFunc()
+		}
+
+		lg.Info("querying Service for ingress endpoint",
+			zap.String("namespace", namespace),
+			zap.String("service-name", svcName),
+		)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+		so, err := c.
+			CoreV1().
+			Services(namespace).
+			Get(ctx, svcName, meta_v1.GetOptions{})
+		cancel()
+		if err != nil {
+			lg.Warn("failed to get Service; retrying", zap.Error(err))
+			if k8s_errors.IsNotFound(err) {
+				return "", "", false, nil
+			}
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		lg.Info(
+			"Service has been linked to LoadBalancer",
+			zap.String("load-balancer", fmt.Sprintf("%+v", so.Status.LoadBalancer)),
+		)
+		for _, ing := range so.Status.LoadBalancer.Ingress {
+			lg.Info(
+				"Service has been linked to LoadBalancer.Ingress",
+				zap.String("ingress", fmt.Sprintf("%+v", ing)),
+			)
+			hostName = ing.Hostname
+			break
+		}
+
+		if hostName != "" {
+			lg.Info("found LoadBalancer host name", zap.String("host-name", hostName))
+			break
+		}
+	}
+
+	if hostName == "" {
+		return "", "", true, errors.New("failed to find LoadBalancer host name")
+	}
+
+	// TODO: find better way to find out the NLB/ELB name
+	ss := strings.Split(hostName, ".")[0]
+	ss = strings.Replace(ss, "-", "/", -1)
+	elbARN = fmt.Sprintf(
+		"arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
+		region,
+		accountID,
+		ss,
+	)
+	lg.Info("found LoadBalancer ELB ARN", zap.String("elb-arn", elbARN))
+
+	return hostName, elbARN, true, nil
 }

k8s-tester/nlb-hello-world/cmd/k8s-tester-nlb-hello-world/main.go

@@ -8,7 +8,10 @@ import (
 
 	"github.com/aws/aws-k8s-tester/client"
 	nlb_hello_world "github.com/aws/aws-k8s-tester/k8s-tester/nlb-hello-world"
+	aws_v1 "github.com/aws/aws-k8s-tester/utils/aws/v1"
 	"github.com/aws/aws-k8s-tester/utils/log"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 )
@@ -55,6 +58,8 @@ func main() {
 }
 
 var (
+	partition              string
+	region                 string
 	deploymentNodeSelector string
 	deploymentReplicas     int32
 )
@@ -65,8 +70,12 @@ func newApply() *cobra.Command {
 		Short: "Apply tests",
 		Run:   createApplyFunc,
 	}
+
+	cmd.PersistentFlags().StringVar(&partition, "partition", "aws", "partition for AWS region")
+	cmd.PersistentFlags().StringVar(&region, "region", "", "region for ELB resource")
 	cmd.PersistentFlags().StringVar(&deploymentNodeSelector, "deployment-node-selector", "", "map of deployment node selector, must be valid JSON format")
 	cmd.PersistentFlags().Int32Var(&deploymentReplicas, "deployment-replicas", nlb_hello_world.DefaultDeploymentReplicas, "number of deployment replicas")
+
 	return cmd
 }
 
@@ -82,6 +91,17 @@ func createApplyFunc(cmd *cobra.Command, args []string) {
 		panic(fmt.Errorf("failed to parse %q (%v)", deploymentNodeSelector, err))
 	}
 
+	awsCfg := aws_v1.Config{
+		Logger:        lg,
+		DebugAPICalls: logLevel == "debug",
+		Partition:     partition,
+		Region:        region,
+	}
+	awsSession, stsOutput, _, err := aws_v1.New(&awsCfg)
+	if err != nil {
+		panic(err)
+	}
+
 	cfg := nlb_hello_world.Config{
 		EnablePrompt: enablePrompt,
 		Logger:       lg,
@@ -92,6 +112,11 @@ func createApplyFunc(cmd *cobra.Command, args []string) {
 			KubectlPath:    kubectlPath,
 			KubeConfigPath: kubeConfigPath,
 		},
+		ELB2API: elbv2.New(awsSession),
+
+		AccountID: aws.StringValue(stsOutput.Account),
+		Region:    region,
+
 		DeploymentNodeSelector: nodeSelector,
 		DeploymentReplicas:     deploymentReplicas,
 	}

k8s-tester/nlb-hello-world/go.mod

@@ -6,12 +6,13 @@ require (
 	github.com/aws/aws-k8s-tester/client v0.0.0-00010101000000-000000000000
 	github.com/aws/aws-k8s-tester/k8s-tester/tester v0.0.0-00010101000000-000000000000
 	github.com/aws/aws-k8s-tester/utils v0.0.0-00010101000000-000000000000
+	github.com/aws/aws-sdk-go v1.38.50
 	github.com/manifoldco/promptui v0.8.0
 	github.com/spf13/cobra v1.1.3
 	go.uber.org/zap v1.17.0
-	k8s.io/api v0.21.0
-	k8s.io/apimachinery v0.21.0
-	k8s.io/client-go v0.21.0
+	k8s.io/api v0.21.1
+	k8s.io/apimachinery v0.21.1
+	k8s.io/client-go v0.21.1
 	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10
 )
 

k8s-tester/nlb-hello-world/go.sum

@@ -41,8 +41,9 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.38.21 h1:D08DXWI4QRaawLaW+OtsIEClOI90I6eheJs1GwXTQVI=
 github.com/aws/aws-sdk-go v1.38.21/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.38.50 h1:9+dEpZbgjBMeoOes6QfZMC87uDMwM8Lw4E79L0/rPZI=
+github.com/aws/aws-sdk-go v1.38.50/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -583,12 +584,15 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
 k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
-k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA=
+k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
+k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
 k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
-k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag=
+k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
+k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
 k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
+k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
+k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=

k8s-tester/nlb-hello-world/tester.go

@@ -14,7 +14,9 @@ import (
 
 	"github.com/aws/aws-k8s-tester/client"
 	k8s_tester "github.com/aws/aws-k8s-tester/k8s-tester/tester"
+	aws_v1_elb "github.com/aws/aws-k8s-tester/utils/aws/v1/elb"
 	"github.com/aws/aws-k8s-tester/utils/http"
+	"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
 	"github.com/manifoldco/promptui"
 	"go.uber.org/zap"
 	apps_v1 "k8s.io/api/apps/v1"
@@ -35,6 +37,10 @@ type Config struct {
 
 	ClientConfig *client.Config
 
+	ELB2API   elbv2iface.ELBV2API
+	AccountID string
+	Region    string
+
 	// Namespace to create test resources.
 	Namespace string `json:"namespace"`
 
@@ -121,6 +127,23 @@ func (ts *tester) Delete() error {
 
 	var errs []string
 
+	// get ELB ARN before deleting the service
+	_, elbARN, exists, err := client.FindServiceIngressHostname(
+		ts.cfg.Logger,
+		ts.cli,
+		ts.cfg.Namespace,
+		serviceName,
+		ts.cfg.Stopc,
+		3*time.Minute,
+		ts.cfg.AccountID,
+		ts.cfg.Region,
+	)
+	if err != nil {
+		if exists { // maybe already deleted from previous run
+			errs = append(errs, fmt.Sprintf("ELB exists but failed to find ingress ELB ARN (%v)", err))
+		}
+	}
+
 	if err := client.DeleteService(
 		ts.cfg.Logger,
 		ts.cli,
@@ -143,6 +166,24 @@ func (ts *tester) Delete() error {
 	ts.cfg.Logger.Info("wait for a minute after deleting Deployment")
 	time.Sleep(time.Minute)
 
+	/*
+	  proactively delete ELB resource
+	  ref. https://github.com/aws/aws-k8s-tester/blob/v1.5.9/eks/nlb-hello-world/nlb-hello-world.go#L135-L154
+
+	  # NLB tags
+	  kubernetes.io/service-name
+	  leegyuho-test-prod-nlb-hello-world/hello-world-service
+	  kubernetes.io/cluster/leegyuho-test-prod
+	  owned
+	*/
+	if err := aws_v1_elb.DeleteELBv2(
+		ts.cfg.Logger,
+		ts.cfg.ELB2API,
+		elbARN,
+	); err != nil {
+		errs = append(errs, fmt.Sprintf("failed to delete ELB (%v)", err))
+	}
+
 	if err := client.DeleteNamespaceAndWait(
 		ts.cfg.Logger,
 		ts.cli,
@@ -344,7 +385,7 @@ func (ts *tester) createService() error {
 	return nil
 }
 
-func (ts *tester) checkService() error {
+func (ts *tester) checkService() (err error) {
 	queryFunc := func() {
 		args := []string{
 			ts.cfg.ClientConfig.KubectlPath,
@@ -366,33 +407,23 @@ func (ts *tester) checkService() error {
 		}
 	}
 
-	hostName, err := client.WaitForServiceIngressHostname(
+	hostName, elbARN, err := client.WaitForServiceIngressHostname(
 		ts.cfg.Logger,
 		ts.cli,
 		ts.cfg.Namespace,
 		serviceName,
 		ts.cfg.Stopc,
 		3*time.Minute,
+		ts.cfg.AccountID,
+		ts.cfg.Region,
 		client.WithQueryFunc(queryFunc),
 	)
 	if err != nil {
 		return err
 	}
-
-	// TODO: is there any better way to find out the NLB name?
-	nlbName := strings.Split(hostName, "-")[0]
-	ss := strings.Split(hostName, ".")[0]
-	ss = strings.Replace(ss, "-", "/", -1)
-	nlbARN := fmt.Sprintf(
-		"arn:aws:elasticloadbalancing:%s:%s:loadbalancer/net/%s",
-		"region",
-		"account-id",
-		ss,
-	)
 	appURL := "http://" + hostName
 
-	fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", nlbARN)
-	fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world Name: %s\n", nlbName)
+	fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", elbARN)
 	fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world URL: %s\n\n", appURL)
 
 	ts.cfg.Logger.Info("waiting before testing hello-world Service")
@@ -425,8 +456,7 @@ func (ts *tester) checkService() error {
 		ts.cfg.Logger.Warn("unexpected hello-world Service output; retrying")
 	}
 
-	fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", nlbARN)
-	fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world Name: %s\n", nlbName)
+	fmt.Fprintf(ts.cfg.LogWriter, "\nNLB hello-world ARN: %s\n", elbARN)
 	fmt.Fprintf(ts.cfg.LogWriter, "NLB hello-world URL: %s\n\n", appURL)
 
 	if !htmlChecked {