GIT-138: add example of multi cluster test workflow

README.md

@@ -1,5 +1,7 @@
 # E2E Framework
 
+[![godoc](https://pkg.go.dev/badge/github.com/sigs.k8s.io/e2e-framework)](https://pkg.go.dev/sigs.k8s.io/e2e-framework)
+
 A Go framework for end-to-end testing of components running in Kubernetes clusters.
 
 The primary goal of this project is to provide a `go test`(able)

examples/multi_cluster/README.md

@@ -0,0 +1,26 @@
+# Multi Cluster Test Runs
+
+This directory contains the example of how to run tests on multiple `kind` clusters at the same time using the same `env.Environment`
+
+## What does this test do ?
+
+1. Create two clusters
+   1. One with prefix cluster-one
+   2. One with prefix cluster-two
+2. Install A sample helm chart on both clusters
+3. Run an assessment to check if the chart has successfully been deployed by checking the pod status
+4. Teardown the Test Environments
+
+# Run Tests
+
+These test cases can be executed using the normal `go test` command by passing the right arguments
+
+```bash
+go test -v .
+```
+
+```bash
+go test -c -o multi-cluster.test .
+
+./multi-cluster.test --v 4
+```

examples/multi_cluster/deployment_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package multi_cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+	"sigs.k8s.io/e2e-framework/third_party/helm"
+)
+
+var curDir, _ = os.Getwd()
+
+func checkPodStatus(t *testing.T, kubeConfig string, clusterName string) {
+	t.Helper()
+	client, err := klient.NewWithKubeConfigFile(kubeConfig)
+	if err != nil {
+		t.Errorf("ran into an error trying to create a client for Cluster %s", clusterName)
+	}
+	deployment := &appsv1.Deployment{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "example",
+			Namespace: "default",
+		},
+		Spec: appsv1.DeploymentSpec{},
+	}
+	err = wait.For(conditions.New(client.Resources()).ResourceScaled(deployment, func(object k8s.Object) int32 {
+		return object.(*appsv1.Deployment).Status.ReadyReplicas
+	}, 1))
+	if err != nil {
+		t.Fatal("failed waiting for the Deployment to reach a ready state")
+	}
+}
+
+func TestScenarioOne(t *testing.T) {
+	feature := features.New("Scenario One").
+		Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			for _, clusterName := range clusterNames {
+				cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterName)
+				if !ok {
+					t.Fatalf("Failed to extract kind cluster %s from context", clusterName)
+				}
+				manager := helm.New(cluster.GetKubeconfig())
+				err := manager.RunInstall(helm.WithName("example"), helm.WithNamespace("default"), helm.WithChart(filepath.Join(curDir, "testdata", "example_chart")), helm.WithWait(), helm.WithTimeout("10m"))
+				if err != nil {
+					t.Fatal("failed to invoke helm install operation due to an error", err)
+				}
+			}
+			return ctx
+		}).
+		Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[0]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[0])
+			if !ok {
+				t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[0])
+			}
+			checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[0])
+			return ctx
+		}).
+		Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[1]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[1])
+			if !ok {
+				t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[1])
+			}
+			checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[1])
+			return ctx
+		}).
+		Feature()
+
+	testEnv.Test(t, feature)
+}

examples/multi_cluster/main_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package multi_cluster
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+var (
+	testEnv      env.Environment
+	clusterNames []string
+)
+
+func TestMain(m *testing.M) {
+	cfg, _ := envconf.NewFromFlags()
+	testEnv = env.NewWithConfig(cfg)
+
+	clusterNames = []string{
+		envconf.RandomName("cluster-one", 16),
+		envconf.RandomName("cluster-two", 16),
+	}
+
+	testEnv.Setup(
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			var err error
+			for _, cluster := range clusterNames {
+				ctx, err = envfuncs.CreateKindCluster(cluster)(ctx, config)
+				if err != nil {
+					return ctx, err
+				}
+			}
+			return ctx, nil
+		},
+	).Finish(
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			var err error
+			for _, cluster := range clusterNames {
+				ctx, err = envfuncs.DestroyKindCluster(cluster)(ctx, config)
+				if err != nil {
+					return ctx, err
+				}
+			}
+			return ctx, nil
+		},
+	)
+
+	os.Exit(testEnv.Run(m))
+}

examples/multi_cluster/testdata/example_chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

examples/multi_cluster/testdata/example_chart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: example
+description: An example Helm chart for Kubernetes e2e-framework
+type: application
+version: 0.1.0
+appVersion: "1.16.0"

examples/multi_cluster/testdata/example_chart/templates/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}
+    spec:
+      automountServiceAccountToken: true
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+

examples/multi_cluster/testdata/example_chart/templates/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ .Release.Name }}

examples/multi_cluster/testdata/example_chart/templates/tests/test-connection.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ .Release.Name }}-test-connection"
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ .Release.Name }}:{{ .Values.service.port }}']
+  restartPolicy: Never

examples/multi_cluster/testdata/example_chart/values.yaml

@@ -0,0 +1,11 @@
+replicaCount: 1
+
+
+image:
+  repository: nginx
+  pullPolicy: IfNotPresent
+  tag: ""
+
+service:
+  type: ClusterIP
+  port: 80

pkg/envfuncs/kind_funcs.go

@@ -33,6 +33,17 @@ import (
 
 type kindContextKey string
 
+// GetKindClusterFromContext helps extract the kind.Cluster object from the context.
+// This can be used to setup and run tests of multi cluster kind.
+func GetKindClusterFromContext(ctx context.Context, clusterName string) (*kind.Cluster, bool) {
+	kindCluster := ctx.Value(kindContextKey(clusterName))
+	if kindCluster == nil {
+		return nil, false
+	}
+	cluster, ok := kindCluster.(*kind.Cluster)
+	return cluster, ok
+}
+
 // CreateKindCluster returns an env.Func that is used to
 // create a kind cluster that is then injected in the context
 // using the name as a key.