test: e2e test framework backbone

.gitignore

@@ -20,3 +20,4 @@
 
 # binary output
 _output/
+hack/tools/bin/

Makefile

@@ -6,6 +6,34 @@ INIT_IMAGE_NAME := proxy-init
 PROXY_IMAGE_TAG := $(REGISTRY)/$(PROXY_IMAGE_NAME):$(IMAGE_VERSION)
 INIT_IMAGE_TAG := $(REGISTRY)/$(INIT_IMAGE_NAME):$(IMAGE_VERSION)
 
+
+# Directories
+ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+BIN_DIR := $(abspath $(ROOT_DIR)/bin)
+TOOLS_DIR := hack/tools
+TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
+
+# Binaries
+E2E_TEST_BIN := e2e.test
+E2E_TEST := $(BIN_DIR)/$(E2E_TEST_BIN)
+
+GINKGO_VER := v1.16.2
+GINKGO_BIN := ginkgo
+GINKGO := $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER)
+
+# Scripts:
+GO_INSTALL = ./hack/go_install.sh
+
+# Ginkgo configurations
+GINKGO_FOCUS ?=
+GINKGO_SKIP ?=
+GINKGO_NODES ?= 3
+GINKGO_NO_COLOR ?= false
+GINKGO_ARGS ?=
+
+# E2E configurations
+E2E_ARGS ?=
+
 build-proxy:
 	CGO_ENABLED=0 GOOS=linux go build -a -o _output/proxy ./cmd/proxy
 
@@ -98,3 +126,28 @@ endif
 # Install cert manager in the cluster
 install-cert-manager:
 	kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.2.0/cert-manager.yaml
+
+$(E2E_TEST):
+	go test -c ./test/e2e -o $(E2E_TEST)
+
+$(GINKGO):
+	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/onsi/ginkgo/ginkgo $(GINKGO_BIN) $(GINKGO_VER)
+
+.PHONY: test-e2e-run
+test-e2e-run: $(E2E_TEST) $(GINKGO)
+	$(GINKGO) -v -trace \
+		-focus="$(GINKGO_FOCUS)" \
+		-skip="$(GINKGO_SKIP)" \
+		-nodes=$(GINKGO_NODES) \
+		-noColor=$(GINKGO_NO_COLOR) \
+		$(E2E_TEST) -- $(E2E_ARGS)
+
+# TODO(chewong): include cluster creation and component installation
+.PHONY: test-e2e
+test-e2e:
+	@echo "no op"
+	$(MAKE) test-e2e-run
+
+.PHONY: clean
+clean:
+	@rm -rf bin/

go.mod

@@ -4,9 +4,42 @@ go 1.16
 
 require (
 	github.com/Azure/go-autorest/autorest v0.11.1
-	gopkg.in/yaml.v2 v2.3.0
+	github.com/onsi/ginkgo v1.16.2
+	github.com/onsi/gomega v1.11.0
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.20.2
 	k8s.io/apimachinery v0.20.2
+	k8s.io/client-go v0.20.2
 	k8s.io/klog/v2 v2.8.0
+	k8s.io/kubernetes v1.20.2
 	sigs.k8s.io/controller-runtime v0.8.3
 )
+
+replace (
+	k8s.io/api => k8s.io/api v0.20.2
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.2
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.2
+	k8s.io/apiserver => k8s.io/apiserver v0.20.2
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.2
+	k8s.io/client-go => k8s.io/client-go v0.20.2
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.2
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.2
+	k8s.io/code-generator => k8s.io/code-generator v0.20.2
+	k8s.io/component-base => k8s.io/component-base v0.20.2
+	k8s.io/component-helpers => k8s.io/component-helpers v0.20.2
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.2
+	k8s.io/cri-api => k8s.io/cri-api v0.20.2
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.2
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.2
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.2
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.2
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.2
+	k8s.io/kubectl => k8s.io/kubectl v0.20.2
+	k8s.io/kubelet => k8s.io/kubelet v0.20.2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.2
+	k8s.io/metrics => k8s.io/metrics v0.20.2
+	k8s.io/mount-utils => k8s.io/mount-utils v0.20.2
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.2
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.20.2
+	k8s.io/sample-controller => k8s.io/sample-controller v0.20.2
+)

hack/go_install.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/master/scripts/go_install.sh
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ -z "${1}" ]]; then
+  echo "must provide module as first parameter"
+  exit 1
+fi
+
+if [[ -z "${2}" ]]; then
+  echo "must provide binary name as second parameter"
+  exit 1
+fi
+
+if [[ -z "${3}" ]]; then
+  echo "must provide version as third parameter"
+  exit 1
+fi
+
+if [[ -z "${GOBIN}" ]]; then
+  echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory."
+  exit 1
+fi
+
+tmp_dir=$(mktemp -d -t goinstall_XXXXXXXXXX)
+function clean {
+  rm -rf "${tmp_dir}"
+}
+trap clean EXIT
+
+rm "${GOBIN}/${2}"* || true
+
+cd "${tmp_dir}"
+
+# create a new module in the tmp directory
+go mod init fake/mod
+
+# install the golang module specified as the first argument
+go get -tags tools "${1}@${3}"
+mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}"
+ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}"

test/e2e/e2e.go

@@ -0,0 +1,87 @@
+package e2e
+
+import (
+	"fmt"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+
+	_ "github.com/Azure/aad-pod-managed-identity/test/e2e/webhook"
+)
+
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	c, err := framework.LoadClientset()
+	if err != nil {
+		framework.Failf("error loading clientset: %v", err)
+	}
+
+	// Delete any namespaces except those created by the system. This ensures no
+	// lingering resources are left over from a previous test run.
+	if framework.TestContext.CleanStart {
+		deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */
+			[]string{
+				metav1.NamespaceSystem,
+				metav1.NamespaceDefault,
+				metav1.NamespacePublic,
+				corev1.NamespaceNodeLease,
+			})
+		if err != nil {
+			framework.Failf("error deleting orphaned namespaces: %v", err)
+		}
+
+		if err := framework.WaitForNamespacesDeleted(c, deleted, 5*time.Minute); err != nil {
+			framework.Failf("error deleting orphaned namespaces %v: %v", deleted, err)
+		}
+	}
+
+	// ensure all nodes are schedulable
+	framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+
+	// Ensure all pods are running and ready before starting tests
+	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
+	if err := e2epod.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil {
+		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
+		e2ekubectl.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf)
+		framework.Failf("error waiting for all pods to be running and ready: %v", err)
+	}
+
+	dc := c.DiscoveryClient
+
+	serverVersion, err := dc.ServerVersion()
+	if err != nil {
+		framework.Logf("unexpected server error retrieving version: %v", err)
+	}
+	if serverVersion != nil {
+		framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
+	}
+
+	return nil
+}, func(data []byte) {})
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	framework.Logf("Running AfterSuite actions on all node")
+	framework.RunCleanupActions()
+}, func() {})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(framework.Fail)
+
+	// Run tests through the Ginkgo runner with output to console + JUnit
+	var r []ginkgo.Reporter
+	if framework.TestContext.ReportDir != "" {
+		r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
+	}
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "AAD Pod Managed Identity E2E Test Suite", r)
+}

test/e2e/e2e_test.go

@@ -0,0 +1,30 @@
+package e2e
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
+)
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+	flag.Parse()
+}
+
+func TestMain(m *testing.M) {
+	// Register test flags, then parse flags.
+	handleFlags()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

test/e2e/webhook/webhook.go

@@ -0,0 +1,26 @@
+package webhook
+
+import (
+	"github.com/onsi/ginkgo"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+)
+
+var _ = ginkgo.Describe("Webhook", func() {
+	f := framework.NewDefaultFramework("webhook")
+
+	var (
+		c clientset.Interface
+	)
+
+	ginkgo.BeforeEach(func() {
+		c = f.ClientSet
+	})
+
+	ginkgo.It("list pods", func() {
+		pods, err := e2epod.GetPodsInNamespace(c, "kube-system", nil)
+		framework.ExpectNoError(err)
+		framework.Logf("Number of pods in kube-system: %d", len(pods))
+	})
+})