Merge pull request #11176 from prezha/fix-cleanup-and-timeout

fix flake integration tests: cleanups and timeouts
kubernetes · Apr 23, 2021 · d1b0989 · d1b0989
2 parents 6f97066 + 79ba91c
commit d1b0989
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 15 deletions.
diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
@@ -31,7 +31,7 @@ export GOPATH="$HOME/go"
 export KUBECONFIG="${TEST_HOME}/kubeconfig"
 export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
 
-readonly TIMEOUT=${1:-70m}
+readonly TIMEOUT=${1:-90m}
 
 if [ "$(uname)" != "Darwin" ]; then
   # install lsof for finding none driver procs, psmisc to use pstree in cronjobs
@@ -48,6 +48,9 @@ sudo ./installers/check_install_golang.sh "1.16" "/usr/local" || true
 docker system prune --force --volumes || true
 docker system df || true
 
+# clean up /tmp
+find /tmp -name . -o -prune -exec rm -rf -- {} + >/dev/null 2>&1 || true
+
 echo ">> Starting at $(date)"
 echo ""
 echo "arch:      ${OS_ARCH}"
@@ -163,11 +166,23 @@ if [[ "${zombie_defuncts}" != "" ]]; then
 fi
 
 if type -P virsh; then
-  virsh -c qemu:///system list --all --uuid \
-    | xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}" \
+  sudo virsh -c qemu:///system list --all --uuid \
+    | xargs -I {} sh -c "sudo virsh -c qemu:///system destroy {}; sudo virsh -c qemu:///system undefine {}" \
     || true
   echo ">> virsh VM list after clean up (should be empty):"
-  virsh -c qemu:///system list --all || true
+  sudo virsh -c qemu:///system list --all || true
+
+  for NET in $( sudo virsh -c qemu:///system net-list --all --name ); do
+    if [ "${NET}" != "default" ]; then
+      sudo virsh -c qemu:///system net-destroy "${NET}" || \
+      sudo virsh -c qemu:///system net-undefine "${NET}" || true
+    fi
+  done
+  echo ">> virsh VM networks list after clean up (should have only 'default'):"
+  sudo virsh -c qemu:///system net-list --all || true
+  echo ">> host networks after KVM clean up:"
+  sudo ip link show || true
+  echo
 fi
 
 if type -P vboxmanage; then

diff --git a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh
@@ -72,13 +72,12 @@ function cleanup() {
 	echo -e "\nbefore the cleanup:"
 	overview
 	for DOM in $( sudo virsh list --all --name ); do
-		if sudo virsh destroy "${DOM}"; then
-			if sudo virsh undefine "${DOM}"; then
-				echo "successfully deleted KVM domain:" "${DOM}"
-				continue
-			fi
-			echo "unable to delete KVM domain:" "${DOM}"
+		sudo virsh destroy "${DOM}" || true
+		if sudo virsh undefine "${DOM}"; then
+			echo "successfully deleted KVM domain:" "${DOM}"
+			continue
 		fi
+		echo "unable to delete KVM domain:" "${DOM}"
 	done
 	#for POOL in $( sudo virsh pool-list --all --name ); do  # better, but flag '--name' is not supported for 'virsh pool-list' command on older libvirt versions
 	for POOL in $( sudo virsh pool-list --all | awk 'NR>2 {print $1}' ); do
@@ -92,11 +91,10 @@ function cleanup() {
 	done
 	for NET in $( sudo virsh net-list --all --name ); do
 		if [ "${NET}" != "default" ]; then
-			if sudo virsh net-destroy "${NET}"; then
-				if sudo virsh net-undefine "${NET}"; then
-					echo "successfully deleted KVM network" "${NET}"
-					continue
-				fi
+			sudo virsh net-destroy "${NET}" || true
+			if sudo virsh net-undefine "${NET}"; then
+				echo "successfully deleted KVM network" "${NET}"
+				continue
 			fi
 			echo "unable to delete KVM network" "${NET}"
 		fi
@@ -113,6 +111,9 @@ function cleanup() {
 	done
 	echo -e "\nafter the cleanup:"
 	overview
+
+	# clean up /tmp
+	find /tmp -name . -o -prune -exec rm -rf -- {} + >/dev/null 2>&1 || true
 }
 
 # Give 15m for Linux-specific cleanup