From f02313ceacf298c3450c9c1a4ec0620356b2b5cc Mon Sep 17 00:00:00 2001 From: Quique Llorente Date: Thu, 26 Sep 2019 10:51:28 +0200 Subject: [PATCH 1/3] DNM: Run the e2e regression at mutiple nodes in parallel to check stability Signed-off-by: Quique Llorente --- Makefile | 2 +- automation/check-patch.k8s-1.14.6-1.mounts | 1 + automation/check-patch.k8s-1.14.6-1.packages | 1 + automation/check-patch.k8s-1.14.6-1.sh | 1 + automation/check-patch.k8s-1.14.6-2.mounts | 1 + automation/check-patch.k8s-1.14.6-2.packages | 1 + automation/check-patch.k8s-1.14.6-2.sh | 1 + automation/check-patch.k8s-1.14.6-3.mounts | 1 + automation/check-patch.k8s-1.14.6-3.packages | 1 + automation/check-patch.k8s-1.14.6-3.sh | 1 + automation/check-patch.k8s-1.14.6-4.mounts | 1 + automation/check-patch.k8s-1.14.6-4.packages | 1 + automation/check-patch.k8s-1.14.6-4.sh | 1 + automation/check-patch.k8s-1.14.6-5.mounts | 1 + automation/check-patch.k8s-1.14.6-5.packages | 1 + automation/check-patch.k8s-1.14.6-5.sh | 1 + automation/test.sh | 2 +- stdci.yaml | 5 +++++ test/e2e/default_bridged_network_test.go | 10 ++++++---- test/e2e/nns_conditions_test.go | 6 ++++++ test/e2e/utils.go | 2 +- 21 files changed, 35 insertions(+), 7 deletions(-) create mode 120000 automation/check-patch.k8s-1.14.6-1.mounts create mode 120000 automation/check-patch.k8s-1.14.6-1.packages create mode 120000 automation/check-patch.k8s-1.14.6-1.sh create mode 120000 automation/check-patch.k8s-1.14.6-2.mounts create mode 120000 automation/check-patch.k8s-1.14.6-2.packages create mode 120000 automation/check-patch.k8s-1.14.6-2.sh create mode 120000 automation/check-patch.k8s-1.14.6-3.mounts create mode 120000 automation/check-patch.k8s-1.14.6-3.packages create mode 120000 automation/check-patch.k8s-1.14.6-3.sh create mode 120000 automation/check-patch.k8s-1.14.6-4.mounts create mode 120000 automation/check-patch.k8s-1.14.6-4.packages create mode 120000 automation/check-patch.k8s-1.14.6-4.sh create mode 120000 automation/check-patch.k8s-1.14.6-5.mounts create mode 120000 automation/check-patch.k8s-1.14.6-5.packages create mode 120000 automation/check-patch.k8s-1.14.6-5.sh diff --git a/Makefile b/Makefile index 0a3384bf3..ee540023b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ ifdef UNIT_TEST_EXTRA_ARGS UNIT_TEST_ARGS += $(UNIT_TEST_ARGS) endif -E2E_TEST_ARGS ?= -test.v -test.timeout=20m -ginkgo.v -ginkgo.slowSpecThreshold=60 +E2E_TEST_ARGS ?= -test.v -test.timeout=40m -ginkgo.v -ginkgo.slowSpecThreshold=60 ifdef E2E_TEST_FOCUS E2E_TEST_ARGS += -ginkgo.focus $(E2E_TEST_FOCUS) endif diff --git a/automation/check-patch.k8s-1.14.6-1.mounts b/automation/check-patch.k8s-1.14.6-1.mounts new file mode 120000 index 000000000..c459a2524 --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-1.mounts @@ -0,0 +1 @@ +check-patch.mounts \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-1.packages b/automation/check-patch.k8s-1.14.6-1.packages new file mode 120000 index 000000000..3858aae7d --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-1.packages @@ -0,0 +1 @@ +check-patch.packages \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-1.sh b/automation/check-patch.k8s-1.14.6-1.sh new file mode 120000 index 000000000..a61652ade --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-1.sh @@ -0,0 +1 @@ +check-patch.sh \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-2.mounts b/automation/check-patch.k8s-1.14.6-2.mounts new file mode 120000 index 000000000..c459a2524 --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-2.mounts @@ -0,0 +1 @@ +check-patch.mounts \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-2.packages b/automation/check-patch.k8s-1.14.6-2.packages new file mode 120000 index 000000000..3858aae7d --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-2.packages @@ -0,0 +1 @@ +check-patch.packages \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-2.sh b/automation/check-patch.k8s-1.14.6-2.sh new file mode 120000 index 000000000..a61652ade --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-2.sh @@ -0,0 +1 @@ +check-patch.sh \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-3.mounts b/automation/check-patch.k8s-1.14.6-3.mounts new file mode 120000 index 000000000..c459a2524 --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-3.mounts @@ -0,0 +1 @@ +check-patch.mounts \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-3.packages b/automation/check-patch.k8s-1.14.6-3.packages new file mode 120000 index 000000000..3858aae7d --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-3.packages @@ -0,0 +1 @@ +check-patch.packages \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-3.sh b/automation/check-patch.k8s-1.14.6-3.sh new file mode 120000 index 000000000..a61652ade --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-3.sh @@ -0,0 +1 @@ +check-patch.sh \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-4.mounts b/automation/check-patch.k8s-1.14.6-4.mounts new file mode 120000 index 000000000..c459a2524 --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-4.mounts @@ -0,0 +1 @@ +check-patch.mounts \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-4.packages b/automation/check-patch.k8s-1.14.6-4.packages new file mode 120000 index 000000000..3858aae7d --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-4.packages @@ -0,0 +1 @@ +check-patch.packages \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-4.sh b/automation/check-patch.k8s-1.14.6-4.sh new file mode 120000 index 000000000..a61652ade --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-4.sh @@ -0,0 +1 @@ +check-patch.sh \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-5.mounts b/automation/check-patch.k8s-1.14.6-5.mounts new file mode 120000 index 000000000..c459a2524 --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-5.mounts @@ -0,0 +1 @@ +check-patch.mounts \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-5.packages b/automation/check-patch.k8s-1.14.6-5.packages new file mode 120000 index 000000000..3858aae7d --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-5.packages @@ -0,0 +1 @@ +check-patch.packages \ No newline at end of file diff --git a/automation/check-patch.k8s-1.14.6-5.sh b/automation/check-patch.k8s-1.14.6-5.sh new file mode 120000 index 000000000..a61652ade --- /dev/null +++ b/automation/check-patch.k8s-1.14.6-5.sh @@ -0,0 +1 @@ +check-patch.sh \ No newline at end of file diff --git a/automation/test.sh b/automation/test.sh index 47d955cca..7cc12244c 100755 --- a/automation/test.sh +++ b/automation/test.sh @@ -8,7 +8,7 @@ teardown() { make cluster-down } -export KUBEVIRT_PROVIDER=$TARGET +export KUBEVIRT_PROVIDER=$(echo $TARGET| sed "s/6-.*/6/") # Make sure that the VM is properly shut down on exit trap teardown EXIT SIGINT SIGTERM SIGSTOP diff --git a/stdci.yaml b/stdci.yaml index eaae2902f..d7d394be6 100644 --- a/stdci.yaml +++ b/stdci.yaml @@ -1,5 +1,10 @@ sub-stages: - k8s-1.14.6 + - k8s-1.14.6-1 + - k8s-1.14.6-2 + - k8s-1.14.6-3 + - k8s-1.14.6-4 + - k8s-1.14.6-5 - os-3.11.0 runtime_requirements: diff --git a/test/e2e/default_bridged_network_test.go b/test/e2e/default_bridged_network_test.go index 79eb7440d..416fcdcee 100644 --- a/test/e2e/default_bridged_network_test.go +++ b/test/e2e/default_bridged_network_test.go @@ -20,7 +20,7 @@ import ( nmstatev1alpha1 "github.com/nmstate/kubernetes-nmstate/pkg/apis/nmstate/v1alpha1" ) -var _ = Describe("NodeNetworkConfigurationPolicy default bridged network", func() { +var _ = PDescribe("NodeNetworkConfigurationPolicy default bridged network", func() { createBridgeOnTheDefaultInterface := nmstatev1alpha1.State(`interfaces: - name: brext type: linux-bridge @@ -142,7 +142,9 @@ func dhcpFlag(node string, name string) bool { func nodeReadyConditionStatus(nodeName string) (corev1.ConditionStatus, error) { key := types.NamespacedName{Name: nodeName} node := corev1.Node{} - err := framework.Global.Client.Get(context.TODO(), key, &node) + oneSecondTimeoutCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err := framework.Global.Client.Get(oneSecondTimeoutCtx, key, &node) if err != nil { return "", err } @@ -157,8 +159,8 @@ func nodeReadyConditionStatus(nodeName string) (corev1.ConditionStatus, error) { func waitForNodesReady() { time.Sleep(5 * time.Second) for _, node := range nodes { - Eventually(func() (corev1.ConditionStatus, error) { + EventuallyWithOffset(1, func() (corev1.ConditionStatus, error) { return nodeReadyConditionStatus(node) - }, 60*time.Second, 1*time.Second).Should(Equal(corev1.ConditionTrue)) + }, 30*time.Minute, 10*time.Second).Should(Equal(corev1.ConditionTrue)) } } diff --git a/test/e2e/nns_conditions_test.go b/test/e2e/nns_conditions_test.go index 3d76730b6..80f69f7a2 100644 --- a/test/e2e/nns_conditions_test.go +++ b/test/e2e/nns_conditions_test.go @@ -56,6 +56,12 @@ var _ = Describe("NodeNetworkStateCondition", func() { BeforeEach(func() { updateDesiredState(invalidConfig) }) + AfterEach(func() { + updateDesiredState(br1Absent) + for _, node := range nodes { + interfacesNameForNode(node).ShouldNot(ContainElement("br1")) + } + }) It("should have Failing ConditionType set to true", func() { for _, node := range nodes { checkCondition(node, nmstatev1alpha1.NodeNetworkStateConditionFailing).Should( diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 0a78aeec2..23fdb8749 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -385,7 +385,7 @@ func vlansCardinality(node string, connection string) AsyncAssertion { func bridgeDescription(node string, bridgeName string) AsyncAssertion { return Eventually(func() (string, error) { return run(node, "sudo", "ip", "-d", "link", "show", "type", "bridge", bridgeName) - }, ReadTimeout, ReadTimeout) + }, ReadTimeout, ReadInterval) } func conditionsToYaml(conditions []nmstatev1alpha1.NodeNetworkStateCondition) string { From 2dadbb9fca3f51dbbeaa13d992e0ba6e5411473b Mon Sep 17 00:00:00 2001 From: Petr Horacek Date: Fri, 27 Sep 2019 18:11:55 +0200 Subject: [PATCH 2/3] fix linux bridge cleanup When bridge setup fails, nmstate sometimes leaves its netlink iface behind although it removes the connection. This issue is tracked in https://github.com/nmstate/nmstate/issues/516. Until that gets resolved, we fix it with ugly sed workaround. Signed-off-by: Petr Horacek --- build/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/Dockerfile b/build/Dockerfile index 98851d129..1a7bde543 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,6 +6,8 @@ RUN sudo dnf install -y nmstate iproute && \ # TODO: Delete this line after we update nmstate to include the change # https://github.com/nmstate/nmstate/commit/a4baaff50fef84d6d326977ff647bece1e917a26 RUN sed -i "s/run(timeout=20/run(timeout=60/g" /usr/lib/python3.7/site-packages/libnmstate/netapplier.py +# TODO: Delete this line after nmstate fixes https://github.com/nmstate/nmstate/issues/516 +RUN sed -i "s/nmclient.NM.DeviceType.OVS_INTERFACE,/nmclient.NM.DeviceType.OVS_INTERFACE, nmclient.NM.DeviceType.BRIDGE,/" /usr/lib/python3.7/site-packages/libnmstate/nm/applier.py # Cannot change the binary to nmstate-handler since the name # is taken from the directory name [1] From 7c7907979b2f18184b2090a1f2680466bb3c13a4 Mon Sep 17 00:00:00 2001 From: Petr Horacek Date: Fri, 27 Sep 2019 23:06:23 +0200 Subject: [PATCH 3/3] DNM: enable default again Signed-off-by: Petr Horacek --- test/e2e/default_bridged_network_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/default_bridged_network_test.go b/test/e2e/default_bridged_network_test.go index 416fcdcee..b00f188c7 100644 --- a/test/e2e/default_bridged_network_test.go +++ b/test/e2e/default_bridged_network_test.go @@ -20,7 +20,7 @@ import ( nmstatev1alpha1 "github.com/nmstate/kubernetes-nmstate/pkg/apis/nmstate/v1alpha1" ) -var _ = PDescribe("NodeNetworkConfigurationPolicy default bridged network", func() { +var _ = Describe("NodeNetworkConfigurationPolicy default bridged network", func() { createBridgeOnTheDefaultInterface := nmstatev1alpha1.State(`interfaces: - name: brext type: linux-bridge