From 55b7f80c5ec058d23f85e0f241bcea0253289688 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 9 Jun 2020 09:13:51 -0700 Subject: [PATCH 1/4] disable failing kubnet tests --- .../test_cluster_configs/windows/network_plugin/kubenet.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json b/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json index 53fa23510c..d9a834de18 100644 --- a/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json +++ b/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json @@ -1,5 +1,8 @@ { - "env": {}, + "env": { + "GINKGO_SKIP": "resolve DNS across windows and linux deployments|should expect containers to be recreated after node restart", + "GINKGO_FAIL_FAST": true + }, "options": { "allowedOrchestratorVersions": [ "1.15", From c5dcbf76024eff6b81dafe0a47393f8e82431ced Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 9 Jun 2020 10:40:59 -0700 Subject: [PATCH 2/4] add warning about kubenet --- docs/topics/features.md | 2 +- pkg/api/vlabs/const.go | 4 +++- pkg/api/vlabs/validate.go | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/topics/features.md b/docs/topics/features.md index 351843c11e..e8682f5381 100644 --- a/docs/topics/features.md +++ b/docs/topics/features.md @@ -514,7 +514,7 @@ We are investigating possible risks & mitigations for when VMs are deprovisioned Kubernetes 1.18 introduces alpha support for the ContainerD runtime on Windows Server 2019. This is still a work-in-progress tracked in [kubernetes/enhancements#1001](https://github.com/kubernetes/enhancements/issues/1001). This feature in AKS-Engine is for testing the in-development versions of ContainerD and Kubernetes, and is not for production use. Be sure to review [open issues](https://github.com/azure/aks-engine/issues?q=containerd+label%3Awindows+is%3Aopen) if you want to test or contribute to this effort. -Currently it only supports the `kubenet` networking model, and requires URLs to custom ContainerD and CNI plugin builds. +Currently it requires URLs to custom ContainerD and CNI plugin builds. ### Deploying multi-OS clusters with ContainerD diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go index 4167878e03..2657c1d934 100644 --- a/pkg/api/vlabs/const.go +++ b/pkg/api/vlabs/const.go @@ -95,7 +95,7 @@ const ( var ( // NetworkPluginValues holds the valid values for network plugin implementation - NetworkPluginValues = [...]string{"", "kubenet", "azure", NetworkPluginCilium, NetworkPluginAntrea, "flannel"} + NetworkPluginValues = [...]string{"", NetworkPluginKubenet, "azure", NetworkPluginCilium, NetworkPluginAntrea, "flannel"} // NetworkPolicyValues holds the valid values for a network policy // "azure" and "none" are there for backwards-compatibility @@ -142,6 +142,8 @@ const ( NetworkModeBridge = "bridge" // NetworkModeTransparent is the string expression for transparent network mode config option NetworkModeTransparent = "transparent" + // NetworkPluginKubenet is the string expression for kubenet network plugin config option + NetworkPluginKubenet = "kubenet" ) const ( diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 223bce179f..0f26b01697 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1667,6 +1667,10 @@ func (k *KubernetesConfig) validateNetworkPlugin(hasWindows bool) error { return errors.Errorf("networkPlugin '%s' is not supporting windows agents", networkPlugin) } + if networkPlugin == NetworkPluginKubenet && hasWindows { + log.Warnf("Windows + Kubenet is for development and testing only, not recommended for production") + } + return nil } From b11ae9d1e1e38495c766ef4d9754050acbf3ec65 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 9 Jun 2020 17:19:09 -0700 Subject: [PATCH 3/4] add test for warning --- pkg/api/vlabs/validate_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index ec5f4285c2..38a80b70be 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -906,6 +906,24 @@ func Test_Properties_ValidateNetworkPolicy(t *testing.T) { } } +func ExampleKubernetesConfig_validateNetworkPlugin() { + log.SetOutput(os.Stdout) + log.SetFormatter(&log.TextFormatter{ + DisableColors: true, + DisableTimestamp: true, + }) + cs := getK8sDefaultContainerService(true) + + cs.Properties.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{} + cs.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = NetworkPluginKubenet + if err := cs.Properties.OrchestratorProfile.KubernetesConfig.validateNetworkPlugin(true); err != nil { + fmt.Printf("error in ValidateNetworkPlugin: %s", err) + } + + // Output: + // level=warning msg="Windows + Kubenet is for development and testing only, not recommended for production" +} + func Test_Properties_ValidateNetworkPlugin(t *testing.T) { p := &Properties{} p.OrchestratorProfile = &OrchestratorProfile{} From 945ba1e44999cc0a168f9eefb2e5ff036b3b31f5 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 9 Jun 2020 19:42:27 -0700 Subject: [PATCH 4/4] skip flaky tests if kubenet --- test/e2e/engine/template.go | 5 +++++ test/e2e/kubernetes/kubernetes_test.go | 8 ++++++++ .../windows/network_plugin/kubenet.json | 5 +---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/e2e/engine/template.go b/test/e2e/engine/template.go index dae05cc62a..0b70155b51 100644 --- a/test/e2e/engine/template.go +++ b/test/e2e/engine/template.go @@ -412,6 +412,11 @@ func (e *Engine) HasNetworkPolicy(name string) bool { return strings.Contains(e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy, name) } +// HasNetworkPolicy will return true if the specified network policy is enabled +func (e *Engine) HasNetworkPlugin(name string) bool { + return strings.Contains(e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin, name) +} + // Write will write the cluster definition to disk func (e *Engine) Write() error { json, err := helpers.JSONMarshal(e.ClusterDefinition, false) diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index 3044a5082b..67260520f8 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -1932,6 +1932,10 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu It("should be able to resolve DNS across windows and linux deployments", func() { if eng.HasWindowsAgents() { + if eng.HasNetworkPlugin(api.NetworkPluginKubenet) { + Skip("This tests is not enabled for kubenet CNI on windows") + } + windowsImages, err := eng.GetWindowsTestImages() Expect(err).NotTo(HaveOccurred()) r := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -2102,6 +2106,10 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } } + if eng.HasNetworkPlugin(api.NetworkPluginKubenet) { + Skip("This tests is not enabled for kubenet CNI on windows") + } + windowsImages, err := eng.GetWindowsTestImages() Expect(err).NotTo(HaveOccurred()) r := rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json b/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json index d9a834de18..53fa23510c 100644 --- a/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json +++ b/test/e2e/test_cluster_configs/windows/network_plugin/kubenet.json @@ -1,8 +1,5 @@ { - "env": { - "GINKGO_SKIP": "resolve DNS across windows and linux deployments|should expect containers to be recreated after node restart", - "GINKGO_FAIL_FAST": true - }, + "env": {}, "options": { "allowedOrchestratorVersions": [ "1.15",