From 58c8d06e388d0c0f5dd6f051b68bdfe25a3ac7cc Mon Sep 17 00:00:00 2001
From: Sneha Chhabria <snchh@microsoft.com>
Date: Tue, 20 Jul 2021 16:21:16 -0700
Subject: [PATCH] Update node for kind cluster on CI

This commit increase the node count for kind cluster used in the CI.

Signed-off-by: Sneha Chhabria <snchh@microsoft.com>
---
 .github/workflows/nightly-noinstall.yml |  6 ++++--
 scripts/kind-with-registry.sh           |  4 ++++
 tests/e2e/e2e_http_ingress_test.go      |  3 +++
 tests/framework/common.go               | 26 ++++++++++++++++++++++---
 4 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/nightly-noinstall.yml b/.github/workflows/nightly-noinstall.yml
index bcef4b67e2..0f4543df56 100644
--- a/.github/workflows/nightly-noinstall.yml
+++ b/.github/workflows/nightly-noinstall.yml
@@ -22,16 +22,18 @@ jobs:
           apiVersion: kind.x-k8s.io/v1alpha4
           nodes:
           - role: control-plane
+          - role: worker
             kubeadmConfigPatches:
             - |
-              kind: InitConfiguration
+              kind: JoinConfiguration
               nodeRegistration:
                 kubeletExtraArgs:
                   node-labels: "ingress-ready=true"
             extraPortMappings:
             - containerPort: 80
               hostPort: 80
-              protocol: TCP
+              protocol: TCP          
+          - role: worker
           EOF
 
           cat kind-config.yaml
diff --git a/scripts/kind-with-registry.sh b/scripts/kind-with-registry.sh
index ae12bf52c1..6db6699984 100755
--- a/scripts/kind-with-registry.sh
+++ b/scripts/kind-with-registry.sh
@@ -42,6 +42,10 @@ echo "Registry Host: ${reg_host}"
 cat <<EOF | kind create cluster --name "${KIND_CLUSTER_NAME}" --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
diff --git a/tests/e2e/e2e_http_ingress_test.go b/tests/e2e/e2e_http_ingress_test.go
index 003789cefd..2f4ab13129 100644
--- a/tests/e2e/e2e_http_ingress_test.go
+++ b/tests/e2e/e2e_http_ingress_test.go
@@ -70,6 +70,9 @@ var _ = OSMDescribe("HTTP ingress",
 						"hostPort": map[string]interface{}{
 							"enabled": true,
 						},
+						"nodeSelector": map[string]interface{}{
+							"ingress-ready": "true",
+						},
 						"service": map[string]interface{}{
 							"type": "NodePort",
 						},
diff --git a/tests/framework/common.go b/tests/framework/common.go
index f8176fd056..068eae02e2 100644
--- a/tests/framework/common.go
+++ b/tests/framework/common.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math"
 	"math/big"
@@ -222,7 +223,10 @@ func (td *OsmTestData) InitTestData(t GinkgoTInterface) error {
 			Nodes: []v1alpha4.Node{
 				{
 					Role: v1alpha4.ControlPlaneRole,
-					KubeadmConfigPatches: []string{`kind: InitConfiguration
+				},
+				{
+					Role: v1alpha4.WorkerRole,
+					KubeadmConfigPatches: []string{`kind: JoinConfiguration
 nodeRegistration:
   kubeletExtraArgs:
     node-labels: "ingress-ready=true"`},
@@ -234,10 +238,15 @@ nodeRegistration:
 						},
 					},
 				},
+				{
+					Role: v1alpha4.WorkerRole,
+				},
 			},
 		}
 		if Td.ClusterVersion != "" {
-			clusterConfig.Nodes[0].Image = fmt.Sprintf("kindest/node:%s", td.ClusterVersion)
+			for i := 0; i < len(clusterConfig.Nodes); i++ {
+				clusterConfig.Nodes[i].Image = fmt.Sprintf("kindest/node:%s", td.ClusterVersion)
+			}
 		}
 		if err := td.ClusterProvider.Create(td.ClusterName, cluster.CreateWithV1Alpha4Config(clusterConfig)); err != nil {
 			return errors.Wrap(err, "failed to create kind cluster")
@@ -346,14 +355,25 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to get image data")
 	}
+
+	imageReader, err := ioutil.ReadAll(imageData)
+	if err != nil {
+		return errors.Wrap(err, "failed to read images")
+	}
+
+	reader := bytes.NewReader(imageReader)
 	defer imageData.Close() //nolint: errcheck,gosec
 	nodes, err := td.ClusterProvider.ListNodes(td.ClusterName)
 	if err != nil {
 		return errors.Wrap(err, "failed to list kind nodes")
 	}
+
 	for _, n := range nodes {
 		td.T.Log("Loading images onto node", n)
-		if err := nodeutils.LoadImageArchive(n, imageData); err != nil {
+		if _, err := reader.Seek(0, io.SeekStart); err != nil {
+			return errors.Wrap(err, "failed to reset images")
+		}
+		if err = nodeutils.LoadImageArchive(n, reader); err != nil {
 			return errors.Wrap(err, "failed to load images")
 		}
 	}