This PR modifies the e2e_egress_test to be able to run on Windows n…

…odes Verified locally in an AKS cluster: ``` Ran 1 of 42 Specs in 471.548 seconds SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 41 Skipped --- PASS: TestSuite (471.56s) ``` Signed-off-by: Sotiris Nanopoulos <[email protected]>
openservicemesh · Aug 23, 2021 · 8d0d6f9 · 8d0d6f9
1 parent eea4672
commit 8d0d6f9
Show file tree

Hide file tree

Showing 14 changed files with 95 additions and 29 deletions.
diff --git a/charts/osm/README.md b/charts/osm/README.md
@@ -139,6 +139,7 @@ The following table lists the configurable parameters of the osm chart and their
 | OpenServiceMesh.prometheus.retention.time | string | `"15d"` | Prometheus data retention time |
 | OpenServiceMesh.pspEnabled | bool | `false` | Run OSM with PodSecurityPolicy configured |
 | OpenServiceMesh.sidecarImage | string | `"envoyproxy/envoy-alpine:v1.19.0"` | Envoy sidecar image |
+| OpenServiceMesh.sidecarWindowsImage | string | `"envoyproxy/envoy-windows:v1.19.0"` |  |
 | OpenServiceMesh.tracing.address | string | `""` | Address of the tracing collector service (must contain the namespace). When left empty, this is computed in helper template to "jaeger.<osm-namespace>.svc.cluster.local". Please override for BYO-tracing as documented in tracing.md |
 | OpenServiceMesh.tracing.enable | bool | `false` | Toggles Envoy's tracing functionality on/off for all sidecar proxies in the mesh |
 | OpenServiceMesh.tracing.endpoint | string | `"/api/v2/spans"` | Tracing collector's API path where the spans will be sent to |

diff --git a/charts/osm/crds/config_meshconfig.yaml b/charts/osm/crds/config_meshconfig.yaml
@@ -64,6 +64,11 @@ spec:
                     envoyImage:
                       description: Image for the Envoy sidecar
                       type: string
+                      default: "envoyproxy/envoy-alpine:v1.19.0"
+                    envoyWindowsImage:
+                      description: Image for the Envoy sidecar on Windows workers
+                      type: string
+                      default: "envoyproxy/envoy-windows:v1.19.0"
                     initContainerImage:
                       description: Image for the init container
                       type: string

diff --git a/charts/osm/templates/preset-mesh-config.yaml b/charts/osm/templates/preset-mesh-config.yaml
@@ -10,6 +10,7 @@ data:
         "logLevel": "{{.Values.OpenServiceMesh.envoyLogLevel}}",
         "maxDataPlaneConnections": {{.Values.OpenServiceMesh.maxDataPlaneConnections}},
         "envoyImage": "{{.Values.OpenServiceMesh.sidecarImage}}",
+        "envoyWindowsImage": "{{.Values.OpenServiceMesh.sidecarWindowsImage}}",
         "initContainerImage": "{{ .Values.OpenServiceMesh.image.registry }}/init:{{ .Values.OpenServiceMesh.image.tag }}",
         "configResyncInterval": "{{.Values.OpenServiceMesh.configResyncInterval}}"
       },

diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json
@@ -257,6 +257,15 @@
                         "envoyproxy/envoy-alpine:v1.19.0"
                     ]
                 },
+                "sidecarWindowsImage": {
+                    "$id": "#/properties/OpenServiceMesh/properties/sidecarWindowsImage",
+                    "type": "string",
+                    "title": "The sidecarWindowsImage schema",
+                    "description": "The proxy side car image to run on Windows payloads.",
+                    "examples": [
+                        "envoyproxy/envoy-windows:v1.19.0"
+                    ]
+                },
                 "certificateProvider": {
                     "$id": "#/properties/OpenServiceMesh/properties/certificateProvider",
                     "type": "object",

diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml
@@ -18,6 +18,7 @@ OpenServiceMesh:
   imagePullSecrets: []
   # -- Envoy sidecar image
   sidecarImage: envoyproxy/envoy-alpine:v1.19.0
+  sidecarWindowsImage: envoyproxy/envoy-windows:v1.19.0
 
   #
   # -- OSM controller parameters

diff --git a/pkg/envoy/bootstrap/config.go b/pkg/envoy/bootstrap/config.go
@@ -1,8 +1,6 @@
 package bootstrap
 
 import (
-	"time"
-
 	xds_accesslog_config "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
 	xds_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
 	xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
@@ -13,7 +11,6 @@ import (
 	xds_upstream_http "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	"github.com/golang/protobuf/ptypes"
 	"github.com/golang/protobuf/ptypes/any"
-	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/openservicemesh/osm/pkg/constants"
 	"github.com/openservicemesh/osm/pkg/envoy"
@@ -140,8 +137,7 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
 		StaticResources: &xds_bootstrap.Bootstrap_StaticResources{
 			Clusters: []*xds_cluster.Cluster{
 				{
-					Name:           config.XDSClusterName,
-					ConnectTimeout: durationpb.New(time.Millisecond * 250),
+					Name: config.XDSClusterName,
 					ClusterDiscoveryType: &xds_cluster.Cluster_Type{
 						Type: xds_cluster.Cluster_LOGICAL_DNS,
 					},

diff --git a/pkg/envoy/bootstrap/config_test.go b/pkg/envoy/bootstrap/config_test.go
@@ -58,8 +58,7 @@ node:
   id: foo.bar.co.uk
 static_resources:
   clusters:
-  - connect_timeout: 0.250s
-    load_assignment:
+  - load_assignment:
       cluster_name: osm-controller
       endpoints:
       - lb_endpoints:

diff --git a/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml b/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml
@@ -25,8 +25,7 @@ node:
   id: foo.bar.co.uk
 static_resources:
   clusters:
-  - connect_timeout: 0.250s
-    load_assignment:
+  - load_assignment:
       cluster_name: osm-controller
       endpoints:
       - lb_endpoints:

diff --git a/tests/e2e/e2e_egress_test.go b/tests/e2e/e2e_egress_test.go
@@ -7,6 +7,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/openservicemesh/osm/pkg/constants"
 	. "github.com/openservicemesh/osm/tests/framework"
 )
 
@@ -32,15 +33,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress",
 				Expect(Td.AddNsToMesh(true, sourceNs)).To(Succeed())
 
 				// Get simple Pod definitions for the client
-				svcAccDef, podDef, svcDef, err := Td.SimplePodApp(SimplePodAppDef{
-					Name:      "client",
-					Namespace: sourceNs,
-					Command:   []string{"/bin/bash", "-c", "--"},
-					Args:      []string{"while true; do sleep 30; done;"},
-					Image:     "songrgg/alpine-debug",
-					Ports:     []int{80},
-					OS:        Td.ClusterOS,
-				})
+				svcAccDef, podDef, svcDef, err := Td.GetOSSpecificSleepPod(sourceNs)
 				Expect(err).NotTo(HaveOccurred())
 
 				_, err = Td.CreateServiceAccount(sourceNs, &svcAccDef)
@@ -51,8 +44,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress",
 				Expect(err).NotTo(HaveOccurred())
 
 				// Expect it to be up and running in it's receiver namespace
-				Expect(Td.WaitForPodsRunningReady(sourceNs, 60*time.Second, 1, nil)).To(Succeed())
-
+				Expect(Td.WaitForPodsRunningReady(sourceNs, 240*time.Second, 1, nil)).To(Succeed())
 				protocols := []string{
 					"http://",
 					"https://",
@@ -68,6 +60,12 @@ var _ = OSMDescribe("HTTP and HTTPS Egress",
 					}
 				}
 
+				waitTime := 60 * time.Second
+				if Td.ClusterOS == constants.OSWindows {
+					//TODO(#4027): remove this platform specific loop.
+					waitTime = 5 * 60 * time.Second
+				}
+
 				for _, url := range urls {
 					cond := Td.WaitForRepeatedSuccess(func() bool {
 						result := Td.HTTPRequest(HTTPRequestDef{
@@ -84,7 +82,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress",
 						}
 						Td.T.Logf("%s > REST req succeeded: %d", url, result.StatusCode)
 						return true
-					}, 5, 60*time.Second)
+					}, 5, waitTime)
 					Expect(cond).To(BeTrue())
 				}
 

diff --git a/tests/framework/common.go b/tests/framework/common.go
@@ -149,6 +149,7 @@ func registerFlags(td *OsmTestData) {
 
 	flag.BoolVar(&td.EnableNsMetricTag, "EnableMetricsTag", true, "Enable tagging Namespaces for metrics collection")
 	flag.BoolVar(&td.DeployOnOpenShift, "deployOnOpenShift", false, "Configure tests to run on OpenShift")
+	flag.BoolVar(&td.DeployOnWindowsWorkers, "deployOnWindowsWorkers", false, "Configure tests to run on Windows workers")
 }
 
 // ValidateStringParams validates input string parameters are valid
@@ -212,7 +213,11 @@ func (td *OsmTestData) InitTestData(t GinkgoTInterface) error {
 	td.TestDirName = fmt.Sprintf("test-%d", td.TestID)
 	td.T.Log(color.HiGreenString("> ID for test: %d, Test dir (abs): %s", td.TestID, td.GetTestDirPath()))
 
-	td.ClusterOS = constants.OSLinux
+	if td.DeployOnWindowsWorkers {
+		td.ClusterOS = constants.OSWindows
+	} else {
+		td.ClusterOS = constants.OSLinux
+	}
 
 	// String parameter validation
 	err = td.ValidateStringParams()
@@ -387,7 +392,7 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error {
 	return nil
 }
 
-func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *v1alpha1.MeshConfig) (defaultConfig *v1alpha1.MeshConfig) {
+func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *v1alpha1.MeshConfig, clusterOS string) (defaultConfig *v1alpha1.MeshConfig) {
 	meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled
 	meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode
 	meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{}
@@ -421,9 +426,7 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
 		td.InitialRestartValues = td.GetOsmCtlComponentRestarts()
 
 		meshConfig, _ := Td.GetMeshConfig(Td.OsmNamespace)
-
-		meshConfig = setMeshConfigToDefault(instOpts, meshConfig)
-
+		meshConfig = setMeshConfigToDefault(instOpts, meshConfig, td.ClusterOS)
 		if _, err := Td.UpdateOSMConfig(meshConfig); err != nil {
 			return err
 		}
@@ -528,6 +531,18 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
 		return err
 	}
 
+	// We need to set some values to the mesh config to disable WASM which is not supported on Windows
+	// For testing we have to use the OSM dev image which matches the underlying OS.
+	if td.ClusterOS == constants.OSWindows {
+		meshConfig, _ := Td.GetMeshConfig(Td.OsmNamespace)
+		meshConfig.Spec.FeatureFlags.EnableWASMStats = false
+		meshConfig.Spec.Sidecar.EnvoyWindowsImage = EnvoyOSMWindowsImage
+		_, err = Td.UpdateOSMConfig(meshConfig)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Store current restart values for CTL processes
 	td.InitialRestartValues = td.GetOsmCtlComponentRestarts()
 

diff --git a/tests/framework/common_apps.go b/tests/framework/common_apps.go
@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
+	"github.com/openservicemesh/osm/pkg/constants"
 	"github.com/openservicemesh/osm/pkg/k8s"
 )
 
@@ -472,6 +473,30 @@ func (td *OsmTestData) SimpleDeploymentApp(def SimpleDeploymentAppDef) (corev1.S
 	return serviceAccountDefinition, deploymentDefinition, serviceDefinition, nil
 }
 
+// GetOSSpecificSleepPod returns a simple OS specific busy loop pod.
+func (td *OsmTestData) GetOSSpecificSleepPod(sourceNs string) (corev1.ServiceAccount, corev1.Pod, corev1.Service, error) {
+	if td.ClusterOS == constants.OSWindows {
+		return Td.SimplePodApp(SimplePodAppDef{
+			Name:      "client",
+			Namespace: sourceNs,
+			Command:   []string{"cmd", "/c"},
+			Args:      []string{"FOR /L %N IN () DO ping -n 30 127.0.0.1> nul"},
+			Image:     WindowsNanoserverDockerImage,
+			Ports:     []int{80},
+			OS:        td.ClusterOS,
+		})
+	}
+	return Td.SimplePodApp(SimplePodAppDef{
+		Name:      "client",
+		Namespace: sourceNs,
+		Command:   []string{"/bin/bash", "-c", "--"},
+		Args:      []string{"while true; do sleep 30; done;"},
+		Image:     "songrgg/alpine-debug",
+		Ports:     []int{80},
+		OS:        td.ClusterOS,
+	})
+}
+
 // GetGrafanaPodHandle generic func to forward a grafana pod and returns a handler pointing to the locally forwarded resource
 func (td *OsmTestData) GetGrafanaPodHandle(ns string, grafanaPodName string, port uint16) (*Grafana, error) {
 	dialer, err := k8s.DialerToPod(td.RestConfig, td.Client, grafanaPodName, ns)

diff --git a/tests/framework/common_traffic.go b/tests/framework/common_traffic.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/fatih/color"
 	. "github.com/onsi/ginkgo"
+
+	"github.com/openservicemesh/osm/pkg/constants"
 )
 
 const (
@@ -89,9 +91,13 @@ func (td *OsmTestData) HTTPRequest(ht HTTPRequestDef) HTTPRequestResult {
 	// -s silent progress, -o output to devnull, '-D -' dump headers to "-" (stdout), -i Status code
 	// -I skip body download, '-w StatusCode:%{http_code}' prints Status code label-like for easy parsing
 	// -L follow redirects
-	commandStr := fmt.Sprintf("/usr/bin/curl -s -o /dev/null -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination)
+	var commandStr string
+	if td.ClusterOS == constants.OSWindows {
+		commandStr = fmt.Sprintf("curl.exe -s -o NUL -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination)
+	} else {
+		commandStr = fmt.Sprintf("/usr/bin/curl -s -o /dev/null -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination)
+	}
 	command := strings.Fields(commandStr)
-
 	stdout, stderr, err := td.RunRemote(ht.SourceNs, ht.SourcePod, ht.SourceContainer, command)
 	if err != nil {
 		// Error codes from the execution come through err

diff --git a/tests/framework/constants.go b/tests/framework/constants.go
@@ -66,3 +66,13 @@ const (
 	// ControlPlaneOnly will collect logs only for control plane processes
 	ControlPlaneOnly CollectLogsType = "controlPlaneOnly"
 )
+
+// Windows Specific container images
+const (
+	// EnvoyOSMWindowsImage is Envoy Windows image used for testing.
+	// On Windows until Windows Server 2022 is publicly available we have to rely on this testing images.
+	EnvoyOSMWindowsImage = "openservicemesh/envoy-windows-nanoserver@sha256:94590d10bc8a46c60cd3a3858d80f3d6577d4e9a191fa05c0077f8b3d6002e22"
+
+	// WindowsNanoserverDockerImage is the base Windows image that is compatible with the test cluster.
+	WindowsNanoserverDockerImage = "mcr.microsoft.com/windows/nanoserver/insider:10.0.20348.1"
+)
diff --git a/tests/framework/types.go b/tests/framework/types.go
@@ -72,7 +72,8 @@ type OsmTestData struct {
 
 	ClusterProvider *cluster.Provider // provider, used when kindCluster is used
 
-	DeployOnOpenShift bool // Determines whether to configure tests for OpenShift
+	DeployOnOpenShift      bool // Determines whether to configure tests for OpenShift
+	DeployOnWindowsWorkers bool // Determines whether to configure tests to run on Windows workers
 }
 
 // InstallOSMOpts describes install options for OSM