add condition helper for checking deployment status and methods for w…

…aiting on object lists

Signed-off-by: Chris Randles <[email protected]>

examples/wait_for_resources/README.md

@@ -0,0 +1,86 @@
+# Waiting for Resource Changes
+
+The test harness supports several methods for querying Kubernetes object types and waiting for conditions to be met. This example shows how to create various wait conditions to drive your tests.
+
+## Waiting for a single object
+
+The wait package has built-in with utilities for waiting on Pods, Jobs, and Deployments:
+
+```go
+func TestPodRunning(t *testing.T) {
+	var err error
+	pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "my-pod"}}
+	err = wait.For(conditions.New(client.Resources()).PodRunning(pod), WithImmediate())
+	if err != nil {
+		t.Error(err)
+	}
+}
+```
+
+Additionally, it is easy to wait for changes to any resource type with the `ResourceMatch` method:
+
+```go
+func TestResourceMatch(t *testing.T) {
+    ...
+    deployment := appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "deploy-name"}}
+	err = wait.For(conditions.New(client.Resources()).ResourceMatch(deployment, func(object k8s.Object) bool {
+		d := object.(*appsv1.Deployment)
+		return d.Status.AvailableReplicas == 2 && d.Status.ReadyReplicas == 2
+	}))
+	if err != nil {
+		t.Error(err)
+	}
+	...
+}
+```
+
+## Waiting for a lists of objects
+
+It is common to need to check for the existence of a set of objects by name:
+
+```go
+func TestResourcesFound(t *testing.T) {
+    ...
+	pods := &v1.PodList{
+		Items: []v1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "p9", Namespace: namespace}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "p10", Namespace: namespace}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "p11", Namespace: namespace}},
+		},
+	}
+    // wait for the set of pods to exist
+	err = wait.For(conditions.New(client.Resources()).ResourcesFound(pods))
+	if err != nil {
+		t.Error(err)
+	}
+	...
+}
+```
+
+Or to check for their absence:
+
+```go
+func TestResourcesDeleted(t *testing.T) {
+    ...
+	pods := &v1.PodList{}
+    // wait for 1 pod with the label `"app": "d5"`
+	err = wait.For(conditions.New(client.Resources()).ResourceListN(
+        pods,
+        1, 
+        resources.WithLabelSelector(labels.FormatLabels(map[string]string{"app": "d5"}))),
+    )
+	if err != nil {
+		t.Error(err)
+	}
+	err = client.Resources().Delete(context.Background(), deployment)
+	if err != nil {
+		t.Error(err)
+	}
+    // wait for the set of pods to finish deleting
+	err = wait.For(conditions.New(client.Resources()).ResourcesDeleted(pods))
+	if err != nil {
+		t.Error(err)
+	}
+    ...
+}
+```

examples/wait_for_resources/main_test.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wait_for_resources
+
+import (
+	"os"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+var testenv env.Environment
+
+func TestMain(m *testing.M) {
+	testenv = env.New()
+	kindClusterName := envconf.RandomName("wait-for-resources", 16)
+	namespace := envconf.RandomName("kind-ns", 16)
+	testenv.Setup(
+		envfuncs.CreateKindCluster(kindClusterName),
+		envfuncs.CreateNamespace(namespace),
+	)
+	testenv.Finish(
+		envfuncs.DeleteNamespace(namespace),
+		envfuncs.DestroyKindCluster(kindClusterName),
+	)
+	os.Exit(testenv.Run(m))
+}

examples/wait_for_resources/wait_test.go

@@ -0,0 +1,128 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wait_for_resources
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestWaitForResources(t *testing.T) {
+	depFeature := features.New("appsv1/deployment").WithLabel("env", "dev").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			// create a deployment
+			deployment := newDeployment(cfg.Namespace(), "test-deployment", 10)
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := client.Resources().Create(ctx, deployment); err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).
+		Assess("deployment >=50% available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			dep := appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: cfg.Namespace()},
+			}
+			// wait for the deployment to become at least 50%
+			err = wait.For(conditions.New(client.Resources()).ResourceMatch(&dep, func(object k8s.Object) bool {
+				d := object.(*appsv1.Deployment)
+				return float64(d.Status.ReadyReplicas)/float64(*d.Spec.Replicas) >= 0.50
+			}), wait.WithTimeout(time.Minute*1))
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("deployment availability: %.2f%%", float64(dep.Status.ReadyReplicas)/float64(*dep.Spec.Replicas)*100)
+			return ctx
+		}).
+		Assess("deployment available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			dep := appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: cfg.Namespace()},
+			}
+			// wait for the deployment to finish becoming available
+			err = wait.For(conditions.New(client.Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue), wait.WithTimeout(time.Minute*1))
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).
+		Assess("deployment pod garbage collection", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			// get list of pods
+			var pods v1.PodList
+			err = client.Resources(cfg.Namespace()).List(context.TODO(), &pods, resources.WithLabelSelector(labels.FormatLabels(map[string]string{"app": "wait-for-resources"})))
+			if err != nil {
+				t.Fatal(err)
+			}
+			// delete the deployment
+			dep := appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: cfg.Namespace()},
+			}
+			err = client.Resources(cfg.Namespace()).Delete(context.TODO(), &dep)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// wait for the deployment pods to be deleted
+			err = wait.For(conditions.New(client.Resources()).ResourcesDeleted(&pods), wait.WithTimeout(time.Minute*1))
+			if err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).Feature()
+
+	testenv.Test(t, depFeature)
+}
+
+func newDeployment(namespace string, name string, replicas int32) *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "wait-for-resources"}},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "wait-for-resources"},
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "wait-for-resources"}},
+				Spec:       v1.PodSpec{Containers: []v1.Container{{Name: "nginx", Image: "nginx"}}},
+			},
+		},
+	}
+}

klient/internal/testutil/setup.go

@@ -17,12 +17,15 @@ limitations under the License.
 package testutil
 
 import (
+	"context"
 	"time"
 
-	log "k8s.io/klog/v2"
-
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	log "k8s.io/klog/v2"
 	"sigs.k8s.io/e2e-framework/klient/conf"
 	"sigs.k8s.io/e2e-framework/support/kind"
 )
@@ -57,6 +60,9 @@ func SetupTestCluster(path string) *TestCluster {
 		log.Fatalln("failed to create new Client set for kind cluster", err)
 	}
 	tc.Clientset = clientSet
+	if err := waitForControlPlane(clientSet); err != nil {
+		log.Fatalln("failed to wait for Kind Cluster control-plane components", err)
+	}
 	return tc
 }
 
@@ -73,9 +79,67 @@ func setupKind() (kc *kind.Cluster, err error) {
 	if _, err = kc.Create(); err != nil {
 		return
 	}
-
-	waitPeriod := 10 * time.Second
-	log.Info("Waiting for kind pods to be initialized...")
-	time.Sleep(waitPeriod)
 	return
 }
+
+func waitForControlPlane(c kubernetes.Interface) error {
+	selector, err := metav1.LabelSelectorAsSelector(
+		&metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{Key: "component", Operator: metav1.LabelSelectorOpIn, Values: []string{"etcd", "kube-apiserver", "kube-controller-manager", "kube-scheduler"}},
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+	options := metav1.ListOptions{LabelSelector: selector.String()}
+	log.Info("Waiting for kind control-plane pods to be initialized...")
+	err = wait.Poll(5*time.Second, time.Minute*2,
+		func() (bool, error) {
+			pods, err := c.CoreV1().Pods("kube-system").List(context.TODO(), options)
+			if err != nil {
+				return false, err
+			}
+			running := 0
+			for i := range pods.Items {
+				if pods.Items[i].Status.Phase == v1.PodRunning {
+					running++
+				}
+			}
+			// a kind cluster with one control-plane node will have 4 pods running the core apiserver components
+			return running >= 4, nil
+		})
+	if err != nil {
+		return err
+	}
+
+	selector, err = metav1.LabelSelectorAsSelector(
+		&metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{Key: "k8s-app", Operator: metav1.LabelSelectorOpIn, Values: []string{"kindnet", "kube-dns", "kube-proxy"}},
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+	options = metav1.ListOptions{LabelSelector: selector.String()}
+	log.Info("Waiting for kind networking pods to be initialized...")
+	err = wait.Poll(5*time.Second, time.Minute*2,
+		func() (bool, error) {
+			pods, err := c.CoreV1().Pods("kube-system").List(context.TODO(), options)
+			if err != nil {
+				return false, err
+			}
+			running := 0
+			for i := range pods.Items {
+				if pods.Items[i].Status.Phase == v1.PodRunning {
+					running++
+				}
+			}
+			// a kind cluster with one control-plane node will have 4 k8s-app pods running networking components
+			return running >= 4, nil
+		})
+	return err
+}