From cbeaf912aee3b6e247944ada840df4b771c959a0 Mon Sep 17 00:00:00 2001 From: Sotiris Nanopoulos Date: Mon, 23 Aug 2021 14:43:39 -0700 Subject: [PATCH] feat(windows): Make Egress test pass on Windows This PR modifies the `e2e_egress_test` to be able to run on Windows nodes Verified locally in an AKS cluster: ``` Ran 1 of 42 Specs in 471.548 seconds SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 41 Skipped --- PASS: TestSuite (471.56s) ``` Signed-off-by: Sotiris Nanopoulos --- charts/osm/README.md | 1 + charts/osm/crds/config_meshconfig.yaml | 5 ++++ charts/osm/templates/preset-mesh-config.yaml | 1 + charts/osm/values.schema.json | 9 +++++++ charts/osm/values.yaml | 1 + pkg/envoy/bootstrap/config.go | 6 +---- pkg/envoy/bootstrap/config_test.go | 3 +-- .../expected_envoy_bootstrap_config.yaml | 3 +-- tests/e2e/e2e_egress_test.go | 22 ++++++++-------- tests/framework/common.go | 25 +++++++++++++++---- tests/framework/common_apps.go | 25 +++++++++++++++++++ tests/framework/common_traffic.go | 10 ++++++-- tests/framework/constants.go | 10 ++++++++ tests/framework/types.go | 3 ++- 14 files changed, 95 insertions(+), 29 deletions(-) diff --git a/charts/osm/README.md b/charts/osm/README.md index f7bd239558..8b6aa3b5d1 100644 --- a/charts/osm/README.md +++ b/charts/osm/README.md @@ -139,6 +139,7 @@ The following table lists the configurable parameters of the osm chart and their | OpenServiceMesh.prometheus.retention.time | string | `"15d"` | Prometheus data retention time | | OpenServiceMesh.pspEnabled | bool | `false` | Run OSM with PodSecurityPolicy configured | | OpenServiceMesh.sidecarImage | string | `"envoyproxy/envoy-alpine:v1.19.0"` | Envoy sidecar image | +| OpenServiceMesh.sidecarWindowsImage | string | `"envoyproxy/envoy-windows:v1.19.0"` | | | OpenServiceMesh.tracing.address | string | `""` | Address of the tracing collector service (must contain the namespace). When left empty, this is computed in helper template to "jaeger..svc.cluster.local". Please override for BYO-tracing as documented in tracing.md | | OpenServiceMesh.tracing.enable | bool | `false` | Toggles Envoy's tracing functionality on/off for all sidecar proxies in the mesh | | OpenServiceMesh.tracing.endpoint | string | `"/api/v2/spans"` | Tracing collector's API path where the spans will be sent to | diff --git a/charts/osm/crds/config_meshconfig.yaml b/charts/osm/crds/config_meshconfig.yaml index 5455692eef..c5f90d2eb8 100644 --- a/charts/osm/crds/config_meshconfig.yaml +++ b/charts/osm/crds/config_meshconfig.yaml @@ -64,6 +64,11 @@ spec: envoyImage: description: Image for the Envoy sidecar type: string + default: "envoyproxy/envoy-alpine:v1.19.0" + envoyWindowsImage: + description: Image for the Envoy sidecar on Windows workers + type: string + default: "envoyproxy/envoy-windows:v1.19.0" initContainerImage: description: Image for the init container type: string diff --git a/charts/osm/templates/preset-mesh-config.yaml b/charts/osm/templates/preset-mesh-config.yaml index 2fd6b07a9e..76d50e7473 100644 --- a/charts/osm/templates/preset-mesh-config.yaml +++ b/charts/osm/templates/preset-mesh-config.yaml @@ -10,6 +10,7 @@ data: "logLevel": "{{.Values.OpenServiceMesh.envoyLogLevel}}", "maxDataPlaneConnections": {{.Values.OpenServiceMesh.maxDataPlaneConnections}}, "envoyImage": "{{.Values.OpenServiceMesh.sidecarImage}}", + "envoyWindowsImage": "{{.Values.OpenServiceMesh.sidecarWindowsImage}}", "initContainerImage": "{{ .Values.OpenServiceMesh.image.registry }}/init:{{ .Values.OpenServiceMesh.image.tag }}", "configResyncInterval": "{{.Values.OpenServiceMesh.configResyncInterval}}" }, diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json index ec99332ebc..8490e86897 100644 --- a/charts/osm/values.schema.json +++ b/charts/osm/values.schema.json @@ -257,6 +257,15 @@ "envoyproxy/envoy-alpine:v1.19.0" ] }, + "sidecarWindowsImage": { + "$id": "#/properties/OpenServiceMesh/properties/sidecarWindowsImage", + "type": "string", + "title": "The sidecarWindowsImage schema", + "description": "The proxy side car image to run on Windows payloads.", + "examples": [ + "envoyproxy/envoy-windows:v1.19.0" + ] + }, "certificateProvider": { "$id": "#/properties/OpenServiceMesh/properties/certificateProvider", "type": "object", diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml index 10ce0967df..c5f3240721 100644 --- a/charts/osm/values.yaml +++ b/charts/osm/values.yaml @@ -18,6 +18,7 @@ OpenServiceMesh: imagePullSecrets: [] # -- Envoy sidecar image sidecarImage: envoyproxy/envoy-alpine:v1.19.0 + sidecarWindowsImage: envoyproxy/envoy-windows:v1.19.0 # # -- OSM controller parameters diff --git a/pkg/envoy/bootstrap/config.go b/pkg/envoy/bootstrap/config.go index 8ec41b6ddf..1b66e0a37e 100644 --- a/pkg/envoy/bootstrap/config.go +++ b/pkg/envoy/bootstrap/config.go @@ -1,8 +1,6 @@ package bootstrap import ( - "time" - xds_accesslog_config "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" xds_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -13,7 +11,6 @@ import ( xds_upstream_http "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/any" - "google.golang.org/protobuf/types/known/durationpb" "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" @@ -140,8 +137,7 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) { StaticResources: &xds_bootstrap.Bootstrap_StaticResources{ Clusters: []*xds_cluster.Cluster{ { - Name: config.XDSClusterName, - ConnectTimeout: durationpb.New(time.Millisecond * 250), + Name: config.XDSClusterName, ClusterDiscoveryType: &xds_cluster.Cluster_Type{ Type: xds_cluster.Cluster_LOGICAL_DNS, }, diff --git a/pkg/envoy/bootstrap/config_test.go b/pkg/envoy/bootstrap/config_test.go index 4e00506e6d..3f4f50d2bb 100644 --- a/pkg/envoy/bootstrap/config_test.go +++ b/pkg/envoy/bootstrap/config_test.go @@ -58,8 +58,7 @@ node: id: foo.bar.co.uk static_resources: clusters: - - connect_timeout: 0.250s - load_assignment: + - load_assignment: cluster_name: osm-controller endpoints: - lb_endpoints: diff --git a/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml b/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml index 9fd0158850..e9518350d2 100644 --- a/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml +++ b/pkg/injector/test_fixtures/expected_envoy_bootstrap_config.yaml @@ -25,8 +25,7 @@ node: id: foo.bar.co.uk static_resources: clusters: - - connect_timeout: 0.250s - load_assignment: + - load_assignment: cluster_name: osm-controller endpoints: - lb_endpoints: diff --git a/tests/e2e/e2e_egress_test.go b/tests/e2e/e2e_egress_test.go index 591103d1d1..1cac6734fa 100644 --- a/tests/e2e/e2e_egress_test.go +++ b/tests/e2e/e2e_egress_test.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/openservicemesh/osm/pkg/constants" . "github.com/openservicemesh/osm/tests/framework" ) @@ -32,15 +33,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress", Expect(Td.AddNsToMesh(true, sourceNs)).To(Succeed()) // Get simple Pod definitions for the client - svcAccDef, podDef, svcDef, err := Td.SimplePodApp(SimplePodAppDef{ - Name: "client", - Namespace: sourceNs, - Command: []string{"/bin/bash", "-c", "--"}, - Args: []string{"while true; do sleep 30; done;"}, - Image: "songrgg/alpine-debug", - Ports: []int{80}, - OS: Td.ClusterOS, - }) + svcAccDef, podDef, svcDef, err := Td.GetOSSpecificSleepPod(sourceNs) Expect(err).NotTo(HaveOccurred()) _, err = Td.CreateServiceAccount(sourceNs, &svcAccDef) @@ -51,8 +44,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress", Expect(err).NotTo(HaveOccurred()) // Expect it to be up and running in it's receiver namespace - Expect(Td.WaitForPodsRunningReady(sourceNs, 60*time.Second, 1, nil)).To(Succeed()) - + Expect(Td.WaitForPodsRunningReady(sourceNs, 240*time.Second, 1, nil)).To(Succeed()) protocols := []string{ "http://", "https://", @@ -68,6 +60,12 @@ var _ = OSMDescribe("HTTP and HTTPS Egress", } } + waitTime := 60 * time.Second + if Td.ClusterOS == constants.OSWindows { + //TODO(#4027): Make the timeouts equal on all platforms. + waitTime = 5 * 60 * time.Second + } + for _, url := range urls { cond := Td.WaitForRepeatedSuccess(func() bool { result := Td.HTTPRequest(HTTPRequestDef{ @@ -84,7 +82,7 @@ var _ = OSMDescribe("HTTP and HTTPS Egress", } Td.T.Logf("%s > REST req succeeded: %d", url, result.StatusCode) return true - }, 5, 60*time.Second) + }, 5, waitTime) Expect(cond).To(BeTrue()) } diff --git a/tests/framework/common.go b/tests/framework/common.go index 1af7eb01ea..e6315e8de8 100644 --- a/tests/framework/common.go +++ b/tests/framework/common.go @@ -149,6 +149,7 @@ func registerFlags(td *OsmTestData) { flag.BoolVar(&td.EnableNsMetricTag, "EnableMetricsTag", true, "Enable tagging Namespaces for metrics collection") flag.BoolVar(&td.DeployOnOpenShift, "deployOnOpenShift", false, "Configure tests to run on OpenShift") + flag.BoolVar(&td.DeployOnWindowsWorkers, "deployOnWindowsWorkers", false, "Configure tests to run on Windows workers") } // ValidateStringParams validates input string parameters are valid @@ -212,7 +213,11 @@ func (td *OsmTestData) InitTestData(t GinkgoTInterface) error { td.TestDirName = fmt.Sprintf("test-%d", td.TestID) td.T.Log(color.HiGreenString("> ID for test: %d, Test dir (abs): %s", td.TestID, td.GetTestDirPath())) - td.ClusterOS = constants.OSLinux + if td.DeployOnWindowsWorkers { + td.ClusterOS = constants.OSWindows + } else { + td.ClusterOS = constants.OSLinux + } // String parameter validation err = td.ValidateStringParams() @@ -387,7 +392,7 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error { return nil } -func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *v1alpha1.MeshConfig) (defaultConfig *v1alpha1.MeshConfig) { +func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *v1alpha1.MeshConfig, clusterOS string) (defaultConfig *v1alpha1.MeshConfig) { meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{} @@ -421,9 +426,7 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { td.InitialRestartValues = td.GetOsmCtlComponentRestarts() meshConfig, _ := Td.GetMeshConfig(Td.OsmNamespace) - - meshConfig = setMeshConfigToDefault(instOpts, meshConfig) - + meshConfig = setMeshConfigToDefault(instOpts, meshConfig, td.ClusterOS) if _, err := Td.UpdateOSMConfig(meshConfig); err != nil { return err } @@ -528,6 +531,18 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { return err } + // We need to set some values to the mesh config to disable WASM which is not supported on Windows + // For testing we have to use the OSM dev image which matches the underlying OS. + if td.ClusterOS == constants.OSWindows { + meshConfig, _ := Td.GetMeshConfig(Td.OsmNamespace) + meshConfig.Spec.FeatureFlags.EnableWASMStats = false + meshConfig.Spec.Sidecar.EnvoyWindowsImage = EnvoyOSMWindowsImage + _, err = Td.UpdateOSMConfig(meshConfig) + if err != nil { + return err + } + } + // Store current restart values for CTL processes td.InitialRestartValues = td.GetOsmCtlComponentRestarts() diff --git a/tests/framework/common_apps.go b/tests/framework/common_apps.go index 28e0449c0b..282794b97a 100644 --- a/tests/framework/common_apps.go +++ b/tests/framework/common_apps.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/k8s" ) @@ -472,6 +473,30 @@ func (td *OsmTestData) SimpleDeploymentApp(def SimpleDeploymentAppDef) (corev1.S return serviceAccountDefinition, deploymentDefinition, serviceDefinition, nil } +// GetOSSpecificSleepPod returns a simple OS specific busy loop pod. +func (td *OsmTestData) GetOSSpecificSleepPod(sourceNs string) (corev1.ServiceAccount, corev1.Pod, corev1.Service, error) { + if td.ClusterOS == constants.OSWindows { + return Td.SimplePodApp(SimplePodAppDef{ + Name: "client", + Namespace: sourceNs, + Command: []string{"cmd", "/c"}, + Args: []string{"FOR /L %N IN () DO ping -n 30 127.0.0.1> nul"}, + Image: WindowsNanoserverDockerImage, + Ports: []int{80}, + OS: td.ClusterOS, + }) + } + return Td.SimplePodApp(SimplePodAppDef{ + Name: "client", + Namespace: sourceNs, + Command: []string{"/bin/bash", "-c", "--"}, + Args: []string{"while true; do sleep 30; done;"}, + Image: "songrgg/alpine-debug", + Ports: []int{80}, + OS: td.ClusterOS, + }) +} + // GetGrafanaPodHandle generic func to forward a grafana pod and returns a handler pointing to the locally forwarded resource func (td *OsmTestData) GetGrafanaPodHandle(ns string, grafanaPodName string, port uint16) (*Grafana, error) { dialer, err := k8s.DialerToPod(td.RestConfig, td.Client, grafanaPodName, ns) diff --git a/tests/framework/common_traffic.go b/tests/framework/common_traffic.go index 2ea5914b46..a5472b8aa0 100644 --- a/tests/framework/common_traffic.go +++ b/tests/framework/common_traffic.go @@ -10,6 +10,8 @@ import ( "github.com/fatih/color" . "github.com/onsi/ginkgo" + + "github.com/openservicemesh/osm/pkg/constants" ) const ( @@ -89,9 +91,13 @@ func (td *OsmTestData) HTTPRequest(ht HTTPRequestDef) HTTPRequestResult { // -s silent progress, -o output to devnull, '-D -' dump headers to "-" (stdout), -i Status code // -I skip body download, '-w StatusCode:%{http_code}' prints Status code label-like for easy parsing // -L follow redirects - commandStr := fmt.Sprintf("/usr/bin/curl -s -o /dev/null -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination) + var commandStr string + if td.ClusterOS == constants.OSWindows { + commandStr = fmt.Sprintf("curl.exe -s -o NUL -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination) + } else { + commandStr = fmt.Sprintf("/usr/bin/curl -s -o /dev/null -D - -I -w %s:%%{http_code} -L %s", StatusCodeWord, ht.Destination) + } command := strings.Fields(commandStr) - stdout, stderr, err := td.RunRemote(ht.SourceNs, ht.SourcePod, ht.SourceContainer, command) if err != nil { // Error codes from the execution come through err diff --git a/tests/framework/constants.go b/tests/framework/constants.go index 74acbc9ab8..7824dddaf8 100644 --- a/tests/framework/constants.go +++ b/tests/framework/constants.go @@ -66,3 +66,13 @@ const ( // ControlPlaneOnly will collect logs only for control plane processes ControlPlaneOnly CollectLogsType = "controlPlaneOnly" ) + +// Windows Specific container images +const ( + // EnvoyOSMWindowsImage is Envoy Windows image used for testing. + // On Windows until Windows Server 2022 is publicly available we have to rely on this testing images. + EnvoyOSMWindowsImage = "openservicemesh/envoy-windows-nanoserver@sha256:94590d10bc8a46c60cd3a3858d80f3d6577d4e9a191fa05c0077f8b3d6002e22" + + // WindowsNanoserverDockerImage is the base Windows image that is compatible with the test cluster. + WindowsNanoserverDockerImage = "mcr.microsoft.com/windows/nanoserver/insider:10.0.20348.1" +) diff --git a/tests/framework/types.go b/tests/framework/types.go index 6f007f3f90..95e1e4c7d7 100644 --- a/tests/framework/types.go +++ b/tests/framework/types.go @@ -72,7 +72,8 @@ type OsmTestData struct { ClusterProvider *cluster.Provider // provider, used when kindCluster is used - DeployOnOpenShift bool // Determines whether to configure tests for OpenShift + DeployOnOpenShift bool // Determines whether to configure tests for OpenShift + DeployOnWindowsWorkers bool // Determines whether to configure tests to run on Windows workers } // InstallOSMOpts describes install options for OSM