diff --git a/README.md b/README.md index bb4f11a9..a7cc3a37 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # E2E Framework +[![godoc](https://pkg.go.dev/badge/github.com/sigs.k8s.io/e2e-framework)](https://pkg.go.dev/sigs.k8s.io/e2e-framework) + A Go framework for end-to-end testing of components running in Kubernetes clusters. The primary goal of this project is to provide a `go test`(able) diff --git a/examples/multi_cluster/README.md b/examples/multi_cluster/README.md new file mode 100644 index 00000000..1e0b3455 --- /dev/null +++ b/examples/multi_cluster/README.md @@ -0,0 +1,26 @@ +# Multi Cluster Test Runs + +This directory contains the example of how to run tests on multiple `kind` clusters at the same time using the same `env.Environment` + +## What does this test do ? + +1. Create two clusters + 1. One with prefix cluster-one + 2. One with prefix cluster-two +2. Install A sample helm chart on both clusters +3. Run an assessment to check if the chart has successfully been deployed by checking the pod status +4. Teardown the Test Environments + +# Run Tests + +These test cases can be executed using the normal `go test` command by passing the right arguments + +```bash +go test -v . +``` + +```bash +go test -c -o multi-cluster.test . + +./multi-cluster.test --v 4 +``` \ No newline at end of file diff --git a/examples/multi_cluster/deployment_test.go b/examples/multi_cluster/deployment_test.go new file mode 100644 index 00000000..278a406f --- /dev/null +++ b/examples/multi_cluster/deployment_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multi_cluster + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/pkg/features" + "sigs.k8s.io/e2e-framework/third_party/helm" +) + +var curDir, _ = os.Getwd() + +func checkPodStatus(t *testing.T, kubeConfig string, clusterName string) { + t.Helper() + client, err := klient.NewWithKubeConfigFile(kubeConfig) + if err != nil { + t.Errorf("ran into an error trying to create a client for Cluster %s", clusterName) + } + deployment := &appsv1.Deployment{ + ObjectMeta: v1.ObjectMeta{ + Name: "example", + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{}, + } + err = wait.For(conditions.New(client.Resources()).ResourceScaled(deployment, func(object k8s.Object) int32 { + return object.(*appsv1.Deployment).Status.ReadyReplicas + }, 1)) + if err != nil { + t.Fatal("failed waiting for the Deployment to reach a ready state") + } +} + +func TestScenarioOne(t *testing.T) { + feature := features.New("Scenario One"). + Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { + for _, clusterName := range clusterNames { + cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterName) + if !ok { + t.Fatalf("Failed to extract kind cluster %s from context", clusterName) + } + manager := helm.New(cluster.GetKubeconfig()) + err := manager.RunInstall(helm.WithName("example"), helm.WithNamespace("default"), helm.WithChart(filepath.Join(curDir, "testdata", "example_chart")), helm.WithWait(), helm.WithTimeout("10m")) + if err != nil { + t.Fatal("failed to invoke helm install operation due to an error", err) + } + } + return ctx + }). + Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[0]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { + cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[0]) + if !ok { + t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[0]) + } + checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[0]) + return ctx + }). + Assess(fmt.Sprintf("Deployment is running successfully - %s", clusterNames[1]), func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { + cluster, ok := envfuncs.GetKindClusterFromContext(ctx, clusterNames[1]) + if !ok { + t.Fatalf("Failed to extract kind cluster %s from context", clusterNames[1]) + } + checkPodStatus(t, cluster.GetKubeconfig(), clusterNames[1]) + return ctx + }). + Feature() + + testEnv.Test(t, feature) +} diff --git a/examples/multi_cluster/main_test.go b/examples/multi_cluster/main_test.go new file mode 100644 index 00000000..3f1a5743 --- /dev/null +++ b/examples/multi_cluster/main_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multi_cluster + +import ( + "context" + "os" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" +) + +var ( + testEnv env.Environment + clusterNames []string +) + +func TestMain(m *testing.M) { + cfg, _ := envconf.NewFromFlags() + testEnv = env.NewWithConfig(cfg) + + clusterNames = []string{ + envconf.RandomName("cluster-one", 16), + envconf.RandomName("cluster-two", 16), + } + + testEnv.Setup( + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + var err error + for _, cluster := range clusterNames { + ctx, err = envfuncs.CreateKindCluster(cluster)(ctx, config) + if err != nil { + return ctx, err + } + } + return ctx, nil + }, + ).Finish( + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + var err error + for _, cluster := range clusterNames { + ctx, err = envfuncs.DestroyKindCluster(cluster)(ctx, config) + if err != nil { + return ctx, err + } + } + return ctx, nil + }, + ) + + os.Exit(testEnv.Run(m)) +} diff --git a/examples/multi_cluster/testdata/example_chart/.helmignore b/examples/multi_cluster/testdata/example_chart/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/examples/multi_cluster/testdata/example_chart/Chart.yaml b/examples/multi_cluster/testdata/example_chart/Chart.yaml new file mode 100644 index 00000000..6cc28272 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: example +description: An example Helm chart for Kubernetes e2e-framework +type: application +version: 0.1.0 +appVersion: "1.16.0" diff --git a/examples/multi_cluster/testdata/example_chart/templates/deployment.yaml b/examples/multi_cluster/testdata/example_chart/templates/deployment.yaml new file mode 100644 index 00000000..136929a2 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/templates/deployment.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + automountServiceAccountToken: true + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + diff --git a/examples/multi_cluster/testdata/example_chart/templates/service.yaml b/examples/multi_cluster/testdata/example_chart/templates/service.yaml new file mode 100644 index 00000000..cfdb09e2 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app: {{ .Release.Name }} diff --git a/examples/multi_cluster/testdata/example_chart/templates/tests/test-connection.yaml b/examples/multi_cluster/testdata/example_chart/templates/tests/test-connection.yaml new file mode 100644 index 00000000..1c9cf693 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/templates/tests/test-connection.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-test-connection" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ .Release.Name }}:{{ .Values.service.port }}'] + restartPolicy: Never \ No newline at end of file diff --git a/examples/multi_cluster/testdata/example_chart/values.yaml b/examples/multi_cluster/testdata/example_chart/values.yaml new file mode 100644 index 00000000..256db1b0 --- /dev/null +++ b/examples/multi_cluster/testdata/example_chart/values.yaml @@ -0,0 +1,11 @@ +replicaCount: 1 + + +image: + repository: nginx + pullPolicy: IfNotPresent + tag: "" + +service: + type: ClusterIP + port: 80 diff --git a/pkg/envfuncs/kind_funcs.go b/pkg/envfuncs/kind_funcs.go index a70e563a..02e531ee 100644 --- a/pkg/envfuncs/kind_funcs.go +++ b/pkg/envfuncs/kind_funcs.go @@ -33,6 +33,17 @@ import ( type kindContextKey string +// GetKindClusterFromContext helps extract the kind.Cluster object from the context. +// This can be used to setup and run tests of multi cluster kind. +func GetKindClusterFromContext(ctx context.Context, clusterName string) (*kind.Cluster, bool) { + kindCluster := ctx.Value(kindContextKey(clusterName)) + if kindCluster == nil { + return nil, false + } + cluster, ok := kindCluster.(*kind.Cluster) + return cluster, ok +} + // CreateKindCluster returns an env.Func that is used to // create a kind cluster that is then injected in the context // using the name as a key.