Add new e2e testing framework

pkg/config/config.go

@@ -37,6 +37,7 @@ func init() {
 	flag.StringVar(&Flags.ServiceAccountTokenPath, "service-account-token-path", "", "optionally override the default token path")
 	flag.StringVar(&Flags.MetricsAddr, "metrics-addr", "0.0.0.0:8081", "address to serve Prometheus metrics on")
 	flag.StringVar(&Flags.ProbeAddr, "probe-addr", "0.0.0.0:8080", "address to serve readiness/liveness probes on")
+	flag.StringVar(&Flags.OperatorNS, "operator-namespace", "kube-system", "namespace of the operator's k8s deployment")
 	flag.StringVar(&Flags.OperatorDeployment, "operator-deployment", "app-routing-operator", "name of the operator's k8s deployment")
 	flag.StringVar(&Flags.ClusterUid, "cluster-uid", "", "unique identifier of the cluster the add-on belongs to")
 }
@@ -58,6 +59,7 @@ type Config struct {
 	ConcurrencyWatchdogThres            float64
 	ConcurrencyWatchdogVotes            int
 	DisableOSM                          bool
+	OperatorNS                          string
 	OperatorDeployment                  string
 	ClusterUid                          string
 }

pkg/controller/controller.go

@@ -147,7 +147,7 @@ func NewManagerForRestConfig(conf *config.Config, rc *rest.Config) (ctrl.Manager
 }
 
 func getSelfDeploy(kcs kubernetes.Interface, conf *config.Config, log logr.Logger) (*appsv1.Deployment, error) {
-	deploy, err := kcs.AppsV1().Deployments(conf.NS).Get(context.Background(), conf.OperatorDeployment, metav1.GetOptions{})
+	deploy, err := kcs.AppsV1().Deployments(conf.OperatorNS).Get(context.Background(), conf.OperatorDeployment, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		// It's okay if we don't find the deployment - just skip setting ownership references latter
 		log.Info("self deploy not found")

pkg/controller/controller_test.go

@@ -71,7 +71,7 @@ func TestLogger(t *testing.T) {
 func TestGetSelfDeploy(t *testing.T) {
 	t.Run("deploy exists", func(t *testing.T) {
 		kcs := fake.NewSimpleClientset()
-		conf := &config.Config{NS: "app-routing-system", OperatorDeployment: "operator"}
+		conf := &config.Config{NS: "", OperatorNS: "app-routing-system", OperatorDeployment: "operator"}
 
 		ns := &corev1.Namespace{}
 		ns.Name = conf.NS
@@ -90,7 +90,7 @@ func TestGetSelfDeploy(t *testing.T) {
 
 	t.Run("deploy missing", func(t *testing.T) {
 		kcs := fake.NewSimpleClientset()
-		conf := &config.Config{NS: "app-routing-system", OperatorDeployment: "operator"}
+		conf := &config.Config{NS: "", OperatorNS: "app-routing-system", OperatorDeployment: "operator"}
 
 		self, err := getSelfDeploy(kcs, conf, logr.Discard())
 		require.NoError(t, err)

testing/e2e/.gitignore

@@ -0,0 +1,2 @@
+infra-config.json
+job-app-routing-operator-e2e.log

testing/e2e/clients/acr.go

@@ -78,7 +78,7 @@ func (a *acr) GetId() string {
 	return a.id
 }
 
-func (a *acr) BuildAndPush(ctx context.Context, imageName string) error {
+func (a *acr) BuildAndPush(ctx context.Context, imageName, dockerfilePath string) error {
 	lgr := logger.FromContext(ctx).With("image", imageName, "name", a.name, "resourceGroup", a.resourceGroup, "subscriptionId", a.subscriptionId)
 	ctx = logger.WithContext(ctx, lgr)
 	lgr.Info("starting to build and push image")
@@ -87,7 +87,7 @@ func (a *acr) BuildAndPush(ctx context.Context, imageName string) error {
 	// Ideally, we'd use the sdk to build and push the image but I couldn't get it working.
 	// I matched everything on the az cli but wasn't able to get it working with the sdk.
 	// https://github.com/Azure/azure-cli/blob/5f9a8fa25cc1c980ebe5e034bd419c95a1c578e2/src/azure-cli/azure/cli/command_modules/acr/build.py#L25
-	cmd := exec.Command("az", "acr", "build", "--registry", a.name, "--image", imageName, ".")
+	cmd := exec.Command("az", "acr", "build", "--registry", a.name, "--image", imageName, dockerfilePath)
 	cmd.Stdout = newLogWriter(lgr, "building and pushing acr image: ", nil)
 	cmd.Stderr = newLogWriter(lgr, "building and pushing acr image: ", to.Ptr(slog.LevelError))
 	if err := cmd.Run(); err != nil {

testing/e2e/clients/aks.go

@@ -6,16 +6,25 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"os"
 
 	"github.com/Azure/aks-app-routing-operator/testing/e2e/logger"
 	"github.com/Azure/aks-app-routing-operator/testing/e2e/manifests"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+	"golang.org/x/exp/slices"
+	"golang.org/x/sync/errgroup"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+var (
+	// https://kubernetes.io/docs/concepts/workloads/
+	// more specifically, these are compatible with kubectl rollout status
+	workloadKinds = []string{"Deployment", "StatefulSet", "DaemonSet"}
+)
+
 type aks struct {
 	name, subscriptionId, resourceGroup string
 	id                                  string
@@ -119,43 +128,160 @@ func (a *aks) Deploy(ctx context.Context, objs []client.Object) error {
 	lgr.Info("starting to deploy resources")
 	defer lgr.Info("finished deploying resources")
 
-	cred, err := getAzCred()
+	zip, err := zipManifests(objs)
 	if err != nil {
-		return fmt.Errorf("getting az credentials: %w", err)
+		return fmt.Errorf("zipping manifests: %w", err)
 	}
+	encoded := base64.StdEncoding.EncodeToString(zip)
 
-	client, err := armcontainerservice.NewManagedClustersClient(a.subscriptionId, cred, nil)
-	if err != nil {
-		return fmt.Errorf("creating aks client: %w", err)
+	if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+		Command: to.Ptr("kubectl apply -f manifests/"),
+		Context: &encoded,
+	}, runCommandOpts{}); err != nil {
+		return fmt.Errorf("running kubectl apply: %w", err)
+	}
+
+	if err := a.waitStable(ctx, objs); err != nil {
+		return fmt.Errorf("waiting for resources to be stable: %w", err)
 	}
 
-	// wrap manifests into base64 zip file.
-	// this is specified by the AKS ARM API.
-	// https://github.com/FumingZhang/azure-cli/blob/aefcf3948ed4207bfcf5d53064e5dac8ea8f19ca/src/azure-cli/azure/cli/command_modules/acs/custom.py#L2750
+	return nil
+}
+
+// zipManifests wraps manifests into base64 zip file.
+// this is specified by the AKS ARM API.
+// https://github.com/FumingZhang/azure-cli/blob/aefcf3948ed4207bfcf5d53064e5dac8ea8f19ca/src/azure-cli/azure/cli/command_modules/acs/custom.py#L2750
+func zipManifests(objs []client.Object) ([]byte, error) {
 	b := &bytes.Buffer{}
 	zipWriter := zip.NewWriter(b)
 	for i, obj := range objs {
 		json, err := manifests.MarshalJson(obj)
 		if err != nil {
-			return fmt.Errorf("marshaling json for object: %w", err)
+			return nil, fmt.Errorf("marshaling json for object: %w", err)
 		}
 
 		f, err := zipWriter.Create(fmt.Sprintf("manifests/%d.json", i))
 		if err != nil {
-			return fmt.Errorf("creating zip entry: %w", err)
+			return nil, fmt.Errorf("creating zip entry: %w", err)
 		}
 
 		if _, err := f.Write(json); err != nil {
-			return fmt.Errorf("writing zip entry: %w", err)
+			return nil, fmt.Errorf("writing zip entry: %w", err)
 		}
 	}
 	zipWriter.Close()
-	encoded := base64.StdEncoding.EncodeToString(b.Bytes())
+	return b.Bytes(), nil
+}
 
-	poller, err := client.BeginRunCommand(ctx, a.resourceGroup, a.name, armcontainerservice.RunCommandRequest{
-		Command: to.Ptr("kubectl apply -f manifests/"),
+func (a *aks) Clean(ctx context.Context, objs []client.Object) error {
+	lgr := logger.FromContext(ctx).With("name", a.name, "resourceGroup", a.resourceGroup)
+	ctx = logger.WithContext(ctx, lgr)
+	lgr.Info("starting to clean resources")
+	defer lgr.Info("finished cleaning resources")
+
+	zip, err := zipManifests(objs)
+	if err != nil {
+		return fmt.Errorf("zipping manifests: %w", err)
+	}
+	encoded := base64.StdEncoding.EncodeToString(zip)
+
+	if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+		Command: to.Ptr("kubectl delete -f manifests/"),
 		Context: &encoded,
-	}, nil)
+	}, runCommandOpts{}); err != nil {
+		return fmt.Errorf("running kubectl delete: %w", err)
+	}
+
+	return nil
+}
+
+func (a *aks) waitStable(ctx context.Context, objs []client.Object) error {
+	lgr := logger.FromContext(ctx).With("name", a.name, "resourceGroup", a.resourceGroup)
+	ctx = logger.WithContext(ctx, lgr)
+	lgr.Info("starting to wait for resources to be stable")
+	defer lgr.Info("finished waiting for resources to be stable")
+
+	var eg errgroup.Group
+	for _, obj := range objs {
+		func(obj client.Object) {
+			eg.Go(func() error {
+				kind := obj.GetObjectKind().GroupVersionKind().GroupKind().Kind
+				ns := obj.GetNamespace()
+				if ns == "" {
+					ns = "default"
+				}
+
+				lgr := lgr.With("kind", kind, "name", obj.GetName(), "namespace", ns)
+				lgr.Info("checking stability of " + kind + "/" + obj.GetName())
+
+				switch {
+				case slices.Contains(workloadKinds, kind):
+					lgr.Info("checking rollout status")
+					if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+						Command: to.Ptr(fmt.Sprintf("kubectl rollout status %s/%s -n %s", kind, obj.GetName(), ns)),
+					}, runCommandOpts{}); err != nil {
+						return fmt.Errorf("waiting for %s/%s to be stable: %w", kind, obj.GetName(), err)
+					}
+				case kind == "Pod":
+					lgr.Info("waiting for pod to be ready")
+					if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+						Command: to.Ptr(fmt.Sprintf("kubectl wait --for=condition=Ready pod/%s -n %s", obj.GetName(), ns)),
+					}, runCommandOpts{}); err != nil {
+						return fmt.Errorf("waiting for pod/%s to be stable: %w", obj.GetName(), err)
+					}
+				case kind == "Job":
+					lgr.Info("waiting for job complete")
+					if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+						Command: to.Ptr(fmt.Sprintf("kubectl logs --pod-running-timeout=20s --follow job/%s -n %s", obj.GetName(), ns)),
+					}, runCommandOpts{
+						outputFile: fmt.Sprintf("job-%s.log", obj.GetName()), // output to a file for jobs because jobs are naturally different from other deployment resources in that waiting for "stability" is waiting for them to complete
+					}); err != nil {
+						return fmt.Errorf("waiting for job/%s to complete: %w", obj.GetName(), err)
+					}
+
+					lgr.Info("checking job statuss")
+					if err := a.runCommand(ctx, armcontainerservice.RunCommandRequest{
+						Command: to.Ptr(fmt.Sprintf("kubectl wait --for=condition=complete --timeout=10s job/%s -n %s", obj.GetName(), ns)),
+					}, runCommandOpts{}); err != nil {
+						return fmt.Errorf("waiting for job/%s to complete: %w", obj.GetName(), err)
+					}
+				}
+
+				return nil
+			})
+		}(obj)
+	}
+
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("waiting for resources to be stable: %w", err)
+	}
+
+	return nil
+}
+
+type runCommandOpts struct {
+	// outputFile is the file to write the output of the command to. Useful for saving logs from a job or something similar
+	// where there's lots of logs that are extremely important and shouldn't be muddled up in the rest of the logs.
+	outputFile string
+}
+
+func (a *aks) runCommand(ctx context.Context, request armcontainerservice.RunCommandRequest, opt runCommandOpts) error {
+	lgr := logger.FromContext(ctx).With("name", a.name, "resourceGroup", a.resourceGroup, "command", *request.Command)
+	ctx = logger.WithContext(ctx, lgr)
+	lgr.Info("starting to run command")
+	defer lgr.Info("finished running command")
+
+	cred, err := getAzCred()
+	if err != nil {
+		return fmt.Errorf("getting az credentials: %w", err)
+	}
+
+	client, err := armcontainerservice.NewManagedClustersClient(a.subscriptionId, cred, nil)
+	if err != nil {
+		return fmt.Errorf("creating aks client: %w", err)
+	}
+
+	poller, err := client.BeginRunCommand(ctx, a.resourceGroup, a.name, request, nil)
 	if err != nil {
 		return fmt.Errorf("starting run command: %w", err)
 	}
@@ -165,9 +291,21 @@ func (a *aks) Deploy(ctx context.Context, objs []client.Object) error {
 		return fmt.Errorf("running command: %w", err)
 	}
 
-	lgr.Info("kubectl apply output: " + *result.Properties.Logs)
+	lgr.Info("command output: " + *result.Properties.Logs)
+	if opt.outputFile != "" {
+		outputFile, err := os.Create(opt.outputFile)
+		if err != nil {
+			return fmt.Errorf("creating output file %s: %w", opt.outputFile, err)
+		}
+		defer outputFile.Close()
+
+		_, err = outputFile.WriteString(*result.Properties.Logs)
+		if err != nil {
+			return fmt.Errorf("writing output file %s: %w", opt.outputFile, err)
+		}
+	}
 	if *result.Properties.ExitCode != 0 {
-		return fmt.Errorf("kubectl apply failed with exit code %d", *result.Properties.ExitCode)
+		return fmt.Errorf("command failed with exit code %d", *result.Properties.ExitCode)
 	}
 
 	return nil

testing/e2e/cmd/flags.go

@@ -51,14 +51,8 @@ var (
 
 func setupInfraFileFlag(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&infraFile, infraFileFlag, "./infra-config.json", "file to load infrastructure from")
-	cmd.MarkFlagRequired(infraFileFlag)
 }
 
 var (
 	infraName string
 )
-
-func setupInfraNameFlag(cmd *cobra.Command) {
-	cmd.Flags().StringVar(&infraName, infraNameFlag, "", "infrastructure name that was provisioned")
-	cmd.MarkFlagRequired(infraNameFlag)
-}

testing/e2e/cmd/test.go

@@ -2,22 +2,58 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
 
+	"github.com/Azure/aks-app-routing-operator/testing/e2e/infra"
 	"github.com/Azure/aks-app-routing-operator/testing/e2e/logger"
+	"github.com/Azure/aks-app-routing-operator/testing/e2e/suites"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	setupInfraNameFlag(testCmd)
+	setupInfraFileFlag(testCmd)
 	rootCmd.AddCommand(testCmd)
 }
 
 var testCmd = &cobra.Command{
 	Use:   "test",
 	Short: "Runs e2e tests",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		lgr := logger.FromContext(context.Background())
-		lgr.Info("Hello World from " + infraName)
+		ctx := cmd.Context()
+		lgr := logger.FromContext(ctx)
+
+		file, err := os.Open(infraFile)
+		if err != nil {
+			return fmt.Errorf("opening file: %w", err)
+		}
+		defer file.Close()
+
+		bytes, err := io.ReadAll(file)
+		if err != nil {
+			return fmt.Errorf("reading file: %w", err)
+		}
+
+		var loaded []infra.LoadableProvisioned
+		if err := json.Unmarshal(bytes, &loaded); err != nil {
+			return fmt.Errorf("unmarshalling saved infrastructure: %w", err)
+		}
+
+		provisioned, err := infra.ToProvisioned(loaded)
+		if err != nil {
+			return fmt.Errorf("generating provisioned infrastructure: %w", err)
+		}
+
+		if len(provisioned) != 1 {
+			return fmt.Errorf("expected 1 provisioned infrastructure, got %d", len(provisioned))
+		}
+
+		tests := suites.All()
+		if err := tests.Run(context.Background(), provisioned[0]); err != nil {
+			return logger.Error(lgr, fmt.Errorf("test failed: %w", err))
+		}
 
 		return nil
 	},