From a74e9faf8b8ff256a1e54417d778f20f2a0c656c Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 3 Dec 2018 21:43:53 -0800 Subject: [PATCH 01/22] Make k8s v1.12.3 the default --- Makefile | 2 +- pkg/minikube/constants/constants.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 290d625ab2d9..0c6eba31682f 100755 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ BUILDROOT_BRANCH ?= 2018.05 REGISTRY?=gcr.io/k8s-minikube HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.10.x -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.10.1-1 +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.12.3-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image ISO_VERSION ?= v0.30.0 diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 0b9966dc580e..026ff0398333 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -22,13 +22,13 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" minikubeVersion "k8s.io/minikube/pkg/version" - "time" ) // APIServerPort is the port that the API server should listen on. @@ -125,7 +125,7 @@ const ( var DefaultIsoUrl = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetIsoPath(), minikubeVersion.GetIsoVersion()) var DefaultIsoShaUrl = DefaultIsoUrl + ShaSuffix -var DefaultKubernetesVersion = "v1.10.0" +var DefaultKubernetesVersion = "v1.12.3" var ConfigFilePath = MakeMiniPath("config") var ConfigFile = MakeMiniPath("config", "config.json") From 8482aeb17bfdc09909870d2f0374c3e3ad2ac5c7 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 4 Dec 2018 11:06:34 -0800 Subject: [PATCH 02/22] Update kube-cross to v1.11.1-1 (latest) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0c6eba31682f..43fc7b6bc416 100755 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ BUILDROOT_BRANCH ?= 2018.05 REGISTRY?=gcr.io/k8s-minikube HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.10.x -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.12.3-1 +# NOTE: "latest" as of 2018-12-04. kube-cross images aren't updated as often as Kubernetes +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.11.1-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image ISO_VERSION ?= v0.30.0 From f18e05a8d43a1e6b57e925bfd2a1cd73c6df71cc Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 4 Dec 2018 13:37:31 -0800 Subject: [PATCH 03/22] none cleanup: force kubeadm, clear /etc/kubernetes, and use sudo for kubectl kill --- hack/jenkins/common.sh | 2 +- hack/jenkins/linux_integration_tests_none.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index de6cf8a4a191..7d60ee8d0c56 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -154,7 +154,7 @@ fi if pgrep kubectl; then echo "killing hung kubectl processes ..." ps -afe | grep kubectl | grep -v grep || true - pgrep kubectl | xargs kill || true + pgrep kubectl | ${SUDO_PREFIX} xargs kill || true fi mkdir -p "${TEST_HOME}" diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh index 6adef9e35423..da2255058e36 100755 --- a/hack/jenkins/linux_integration_tests_none.sh +++ b/hack/jenkins/linux_integration_tests_none.sh @@ -36,9 +36,11 @@ export KUBECONFIG="/root/.kube/config" # "none" driver specific cleanup from previous runs. # kubeadm -sudo kubeadm reset || true +sudo kubeadm reset -f || true # Cleanup data directory sudo rm -rf /data/* +# Cleanup old Kubernetes configs +sudo rm -rf /etc/kubernetes/* # Stop any leftover kubelets systemctl is-active --quiet kubelet \ && echo "stopping kubelet" \ From 2a67d65bf50a4bf325ed3d2a3dcd785b9c096826 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 4 Dec 2018 14:11:15 -0800 Subject: [PATCH 04/22] Add tunnels.json and netstat -rn (before & after) to debug output --- hack/jenkins/common.sh | 6 ++++++ hack/jenkins/print-debug-info.sh | 2 ++ 2 files changed, 8 insertions(+) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 390881be3271..46a31a62bff2 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -96,6 +96,12 @@ for stale_dir in "${TEST_ROOT}"/*; do export KUBECONFIG="${stale_dir}/kubeconfig" if [[ -d "${MINIKUBE_HOME}" ]]; then + if [[ -f "${MINIKUBE_HOME/tunnels.json" ]]; then + echo "Stale tunnels.json:" + cat "${MINIKUBE_HOME}/tunnels.json" + echo "Current routes:" + netstat -rn + fi echo "Shutting down stale minikube instance ..." if [[ -w "${MINIKUBE_HOME}" ]]; then "${MINIKUBE_BIN}" delete || true diff --git a/hack/jenkins/print-debug-info.sh b/hack/jenkins/print-debug-info.sh index 8aa2714abd86..ef0838404d93 100755 --- a/hack/jenkins/print-debug-info.sh +++ b/hack/jenkins/print-debug-info.sh @@ -60,6 +60,8 @@ if type -P hdiutil; then hdiutil info | grep -E "/dev/disk[1-9][^s]" fi +netstat -rn + echo "" echo ">>> end print-debug-info" echo "" From 568c4ef889fcb1cdf486bee347b84438bb2596e2 Mon Sep 17 00:00:00 2001 From: Balint Pato Date: Tue, 4 Dec 2018 15:15:54 -0800 Subject: [PATCH 05/22] Add -f inet Co-Authored-By: tstromberg --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 46a31a62bff2..221c75a6b83a 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -100,7 +100,7 @@ for stale_dir in "${TEST_ROOT}"/*; do echo "Stale tunnels.json:" cat "${MINIKUBE_HOME}/tunnels.json" echo "Current routes:" - netstat -rn + netstat -rn -f inet fi echo "Shutting down stale minikube instance ..." if [[ -w "${MINIKUBE_HOME}" ]]; then From 28bfc5ec538793b1954912c87e2b8a85eb2e839f Mon Sep 17 00:00:00 2001 From: Balint Pato Date: Tue, 4 Dec 2018 15:16:05 -0800 Subject: [PATCH 06/22] Update hack/jenkins/print-debug-info.sh Co-Authored-By: tstromberg --- hack/jenkins/print-debug-info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/print-debug-info.sh b/hack/jenkins/print-debug-info.sh index ef0838404d93..d5b256dd590e 100755 --- a/hack/jenkins/print-debug-info.sh +++ b/hack/jenkins/print-debug-info.sh @@ -60,7 +60,7 @@ if type -P hdiutil; then hdiutil info | grep -E "/dev/disk[1-9][^s]" fi -netstat -rn +netstat -rn -f inet echo "" echo ">>> end print-debug-info" From 4b33fd2dc4a2a40ba2666113989be09013b44bf8 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 10:18:33 -0800 Subject: [PATCH 07/22] Add missing } --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 221c75a6b83a..b12e3c096535 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -96,7 +96,7 @@ for stale_dir in "${TEST_ROOT}"/*; do export KUBECONFIG="${stale_dir}/kubeconfig" if [[ -d "${MINIKUBE_HOME}" ]]; then - if [[ -f "${MINIKUBE_HOME/tunnels.json" ]]; then + if [[ -f "${MINIKUBE_HOME}/tunnels.json" ]]; then echo "Stale tunnels.json:" cat "${MINIKUBE_HOME}/tunnels.json" echo "Current routes:" From 82668a7c0d832bb408a6e867dadd3da804e19b36 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 10:49:17 -0800 Subject: [PATCH 08/22] Fix variable expansion to avoid confusing error message --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index b12e3c096535..18c369a1b470 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -90,7 +90,7 @@ fi echo "" echo ">> Cleaning up after previous test runs ..." -for stale_dir in "${TEST_ROOT}"/*; do +for stale_dir in ${TEST_ROOT}/*; do echo "* Cleaning stale test: ${stale_dir}" export MINIKUBE_HOME="${stale_dir}/.minikube" export KUBECONFIG="${stale_dir}/kubeconfig" From d7cc1a2938dbd4b4e8fa50c52791222023fa4430 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 12:02:15 -0800 Subject: [PATCH 09/22] Cleanup stale minikube routes instead of just outputting a route table --- hack/jenkins/common.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 8b9bb6d782f5..482d2b170ee7 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -99,8 +99,6 @@ for stale_dir in ${TEST_ROOT}/*; do if [[ -f "${MINIKUBE_HOME}/tunnels.json" ]]; then echo "Stale tunnels.json:" cat "${MINIKUBE_HOME}/tunnels.json" - echo "Current routes:" - netstat -rn -f inet fi echo "Shutting down stale minikube instance ..." if [[ -w "${MINIKUBE_HOME}" ]]; then @@ -163,6 +161,18 @@ if pgrep kubectl; then pgrep kubectl | xargs kill || true fi +function cleanup_stale_routes() { + stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0) + if [[ "${stale_routes}" != "" ]]; then + echo "WARNING: deleting stale tunnel routes: ${stale_routes}" + for route in ${stale_routes}; do + sudo route -n delete "${route}" || true + done + fi +} + +cleanup_stale_routes || true + mkdir -p "${TEST_HOME}" export MINIKUBE_HOME="${TEST_HOME}/.minikube" export MINIKUBE_WANTREPORTERRORPROMPT=False @@ -193,6 +203,7 @@ fi echo ">> Cleaning up after ourselves ..." ${SUDO_PREFIX}${MINIKUBE_BIN} delete >/dev/null 2>/dev/null || true +cleanup_stale_routes || true ${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true From 0a4080142142c5ae1252ba06dab592333e699838 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 13:18:00 -0800 Subject: [PATCH 10/22] kill hung test commands and improve hung process output --- hack/jenkins/common.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index de6cf8a4a191..a54a5f334d75 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -81,9 +81,12 @@ export E2E_BIN="out/e2e-${OS_ARCH}" chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}" out/docker-machine-driver-* -if pgrep "minikube|e2e-"; then - echo "WARNING: other instances of minikube may be running:" - ps -afe | egrep "minikube|e2e-" +procs=$(pgrep "minikube-${OS_ARCH}|e2e-${OS_ARCH}") +if [[ "${procs}" != "" ]]; then + echo "ERROR: found stale test processes to kill:" + ps -f -p ${procs} || true + kill ${procs} || true + kill -9 ${procs} || true fi # Cleanup stale test outputs. @@ -151,10 +154,11 @@ if [[ "${VM_DRIVER}" == "hyperkit" ]]; then fi fi -if pgrep kubectl; then - echo "killing hung kubectl processes ..." - ps -afe | grep kubectl | grep -v grep || true - pgrep kubectl | xargs kill || true +kprocs=$(pgrep kubectl) +if [[ "${kprocs}" != "" ]]; then + echo "error: killing hung kubectl processes ..." + ps -f -p ${kprocs} true + kill ${kprocs} || true fi mkdir -p "${TEST_HOME}" From 1b155d9952c16310a00ade613d6cf7272bd16ddd Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 13:48:31 -0800 Subject: [PATCH 11/22] add or-trues --- hack/jenkins/common.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index a54a5f334d75..c277b3bb2568 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -80,8 +80,7 @@ export MINIKUBE_BIN="out/minikube-${OS_ARCH}" export E2E_BIN="out/e2e-${OS_ARCH}" chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}" out/docker-machine-driver-* - -procs=$(pgrep "minikube-${OS_ARCH}|e2e-${OS_ARCH}") +procs=$(pgrep "minikube-${OS_ARCH}|e2e-${OS_ARCH}" || true) if [[ "${procs}" != "" ]]; then echo "ERROR: found stale test processes to kill:" ps -f -p ${procs} || true @@ -154,10 +153,10 @@ if [[ "${VM_DRIVER}" == "hyperkit" ]]; then fi fi -kprocs=$(pgrep kubectl) +kprocs=$(pgrep kubectl || true) if [[ "${kprocs}" != "" ]]; then echo "error: killing hung kubectl processes ..." - ps -f -p ${kprocs} true + ps -f -p ${kprocs} || true kill ${kprocs} || true fi From b5707d56e6b4a43b6c0717076c12e34d31451dd2 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 13:49:56 -0800 Subject: [PATCH 12/22] add or-trues --- hack/jenkins/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 482d2b170ee7..f8f3b94aeffc 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -162,7 +162,7 @@ if pgrep kubectl; then fi function cleanup_stale_routes() { - stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0) + stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0 || true) if [[ "${stale_routes}" != "" ]]; then echo "WARNING: deleting stale tunnel routes: ${stale_routes}" for route in ${stale_routes}; do From eaf6c9248d4a2d290b3f0174a75bf7849059c53e Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 15:13:48 -0800 Subject: [PATCH 13/22] Explicitly call tunnel --cleanup, use ip route on Linux --- hack/jenkins/common.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index f8f3b94aeffc..2bda885f32c3 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -96,9 +96,9 @@ for stale_dir in ${TEST_ROOT}/*; do export KUBECONFIG="${stale_dir}/kubeconfig" if [[ -d "${MINIKUBE_HOME}" ]]; then - if [[ -f "${MINIKUBE_HOME}/tunnels.json" ]]; then - echo "Stale tunnels.json:" + if [[ -r "${MINIKUBE_HOME}/tunnels.json" ]]; then cat "${MINIKUBE_HOME}/tunnels.json" + ${MINIKUBE_BIN} tunnel --clean || true fi echo "Shutting down stale minikube instance ..." if [[ -w "${MINIKUBE_HOME}" ]]; then @@ -166,7 +166,11 @@ function cleanup_stale_routes() { if [[ "${stale_routes}" != "" ]]; then echo "WARNING: deleting stale tunnel routes: ${stale_routes}" for route in ${stale_routes}; do - sudo route -n delete "${route}" || true + if [[ "$(uname)" == "Linux" ]]; then + sudo ip route delete "${route}" || true + else + sudo route -n delete "${route}" || true + fi done fi } @@ -202,7 +206,8 @@ else fi echo ">> Cleaning up after ourselves ..." -${SUDO_PREFIX}${MINIKUBE_BIN} delete >/dev/null 2>/dev/null || true +${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --clean >/dev/null 2>/dev/null || true +${SUDO_PREFIX}${MINIKUBE_BIN} delete || true cleanup_stale_routes || true ${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true From 8192620c3f33f39f076dc5cb68b94fcf2e8fbacb Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 15:20:13 -0800 Subject: [PATCH 14/22] tunnel --clean should be tunnel --cleanup --- hack/jenkins/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 2bda885f32c3..80ce3b0659ab 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -98,7 +98,7 @@ for stale_dir in ${TEST_ROOT}/*; do if [[ -d "${MINIKUBE_HOME}" ]]; then if [[ -r "${MINIKUBE_HOME}/tunnels.json" ]]; then cat "${MINIKUBE_HOME}/tunnels.json" - ${MINIKUBE_BIN} tunnel --clean || true + ${MINIKUBE_BIN} tunnel --cleanup || true fi echo "Shutting down stale minikube instance ..." if [[ -w "${MINIKUBE_HOME}" ]]; then @@ -206,7 +206,7 @@ else fi echo ">> Cleaning up after ourselves ..." -${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --clean >/dev/null 2>/dev/null || true +${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup >/dev/null 2>/dev/null || true ${SUDO_PREFIX}${MINIKUBE_BIN} delete || true cleanup_stale_routes || true From cd5d5b33a302fe5cf1a91d1f2391a8a1b8a7ac5d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 15:24:01 -0800 Subject: [PATCH 15/22] Redirect output on delete, leave it on cleanup --- hack/jenkins/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 80ce3b0659ab..eef2e7555464 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -206,8 +206,8 @@ else fi echo ">> Cleaning up after ourselves ..." -${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup >/dev/null 2>/dev/null || true -${SUDO_PREFIX}${MINIKUBE_BIN} delete || true +${SUDO_PREFIX}${MINIKUBE_BIN} tunnel --cleanup || true +${SUDO_PREFIX}${MINIKUBE_BIN} delete >/dev/null 2>/dev/null || true cleanup_stale_routes || true ${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}" || true From fc5256b21e656ec7df21df99f7ccabbdd5b7e9f5 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 17:27:39 -0800 Subject: [PATCH 16/22] Use ip route commands on Linux, as they show the subnet which may be required for deletion --- .../minikube-iso/configs/minikube_defconfig | 1 + hack/jenkins/common.sh | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig index 8a6f8363ca16..c70b9fec76de 100644 --- a/deploy/iso/minikube-iso/configs/minikube_defconfig +++ b/deploy/iso/minikube-iso/configs/minikube_defconfig @@ -53,6 +53,7 @@ BR2_PACKAGE_SYSTEMD_LOGIND=y BR2_PACKAGE_SYSTEMD_MACHINED=y BR2_PACKAGE_SYSTEMD_VCONSOLE=y BR2_PACKAGE_UTIL_LINUX_NSENTER=y +BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y BR2_TARGET_ROOTFS_CPIO_BZIP2=y BR2_TARGET_ROOTFS_ISO9660=y BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg" diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index eef2e7555464..d198318ad2ec 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -162,17 +162,19 @@ if pgrep kubectl; then fi function cleanup_stale_routes() { - stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0 || true) - if [[ "${stale_routes}" != "" ]]; then - echo "WARNING: deleting stale tunnel routes: ${stale_routes}" - for route in ${stale_routes}; do - if [[ "$(uname)" == "Linux" ]]; then - sudo ip route delete "${route}" || true - else - sudo route -n delete "${route}" || true - fi - done + local show="netstat -rn -f inet" + local del="sudo route -n delete" + + if [[ "$(uname)" == "Linux" ]]; then + show="ip route show" + del="sudo ip route delete" fi + + local troutes=$($show | awk '{ print $1 }' | grep 10.96.0.0 || true) + for route in ${troutes}; do + echo "WARNING: deleting stale tunnel route: ${route}" + $del "${route}" || true + done } cleanup_stale_routes || true From 468182238c34e50caa94dd6f5b360769bfbc2a79 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 17:28:19 -0800 Subject: [PATCH 17/22] Revert "Use ip route commands on Linux, as they show the subnet which may be required for deletion" This reverts commit fc5256b21e656ec7df21df99f7ccabbdd5b7e9f5. --- .../minikube-iso/configs/minikube_defconfig | 1 - hack/jenkins/common.sh | 22 +++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/deploy/iso/minikube-iso/configs/minikube_defconfig b/deploy/iso/minikube-iso/configs/minikube_defconfig index c70b9fec76de..8a6f8363ca16 100644 --- a/deploy/iso/minikube-iso/configs/minikube_defconfig +++ b/deploy/iso/minikube-iso/configs/minikube_defconfig @@ -53,7 +53,6 @@ BR2_PACKAGE_SYSTEMD_LOGIND=y BR2_PACKAGE_SYSTEMD_MACHINED=y BR2_PACKAGE_SYSTEMD_VCONSOLE=y BR2_PACKAGE_UTIL_LINUX_NSENTER=y -BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y BR2_TARGET_ROOTFS_CPIO_BZIP2=y BR2_TARGET_ROOTFS_ISO9660=y BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg" diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index d198318ad2ec..eef2e7555464 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -162,19 +162,17 @@ if pgrep kubectl; then fi function cleanup_stale_routes() { - local show="netstat -rn -f inet" - local del="sudo route -n delete" - - if [[ "$(uname)" == "Linux" ]]; then - show="ip route show" - del="sudo ip route delete" + stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0 || true) + if [[ "${stale_routes}" != "" ]]; then + echo "WARNING: deleting stale tunnel routes: ${stale_routes}" + for route in ${stale_routes}; do + if [[ "$(uname)" == "Linux" ]]; then + sudo ip route delete "${route}" || true + else + sudo route -n delete "${route}" || true + fi + done fi - - local troutes=$($show | awk '{ print $1 }' | grep 10.96.0.0 || true) - for route in ${troutes}; do - echo "WARNING: deleting stale tunnel route: ${route}" - $del "${route}" || true - done } cleanup_stale_routes || true From 198c1724c7f53edb375bc19ea4b7b9ee242c9c6d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 4 Dec 2018 13:37:31 -0800 Subject: [PATCH 18/22] none cleanup: force kubeadm, clear /etc/kubernetes, and use sudo for kubectl kill --- hack/jenkins/common.sh | 2 +- hack/jenkins/linux_integration_tests_none.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index eef2e7555464..fe156a3a5c32 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -158,7 +158,7 @@ fi if pgrep kubectl; then echo "killing hung kubectl processes ..." ps -afe | grep kubectl | grep -v grep || true - pgrep kubectl | xargs kill || true + pgrep kubectl | ${SUDO_PREFIX} xargs kill || true fi function cleanup_stale_routes() { diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh index 6adef9e35423..da2255058e36 100755 --- a/hack/jenkins/linux_integration_tests_none.sh +++ b/hack/jenkins/linux_integration_tests_none.sh @@ -36,9 +36,11 @@ export KUBECONFIG="/root/.kube/config" # "none" driver specific cleanup from previous runs. # kubeadm -sudo kubeadm reset || true +sudo kubeadm reset -f || true # Cleanup data directory sudo rm -rf /data/* +# Cleanup old Kubernetes configs +sudo rm -rf /etc/kubernetes/* # Stop any leftover kubelets systemctl is-active --quiet kubelet \ && echo "stopping kubelet" \ From 174a76006368c5558e02beae1be0ecb110163be1 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 3 Dec 2018 21:43:53 -0800 Subject: [PATCH 19/22] Make k8s v1.12.3 the default --- Makefile | 2 +- pkg/minikube/constants/constants.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 290d625ab2d9..0c6eba31682f 100755 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ BUILDROOT_BRANCH ?= 2018.05 REGISTRY?=gcr.io/k8s-minikube HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.10.x -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.10.1-1 +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.12.3-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image ISO_VERSION ?= v0.30.0 diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 0b9966dc580e..026ff0398333 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -22,13 +22,13 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" minikubeVersion "k8s.io/minikube/pkg/version" - "time" ) // APIServerPort is the port that the API server should listen on. @@ -125,7 +125,7 @@ const ( var DefaultIsoUrl = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetIsoPath(), minikubeVersion.GetIsoVersion()) var DefaultIsoShaUrl = DefaultIsoUrl + ShaSuffix -var DefaultKubernetesVersion = "v1.10.0" +var DefaultKubernetesVersion = "v1.12.3" var ConfigFilePath = MakeMiniPath("config") var ConfigFile = MakeMiniPath("config", "config.json") From f643387595f3db08ec9103cf3ebe5f9351ff61b2 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 4 Dec 2018 11:06:34 -0800 Subject: [PATCH 20/22] Update kube-cross to v1.11.1-1 (latest) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0c6eba31682f..43fc7b6bc416 100755 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ BUILDROOT_BRANCH ?= 2018.05 REGISTRY?=gcr.io/k8s-minikube HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.10.x -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.12.3-1 +# NOTE: "latest" as of 2018-12-04. kube-cross images aren't updated as often as Kubernetes +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.11.1-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image ISO_VERSION ?= v0.30.0 From 41fd26a063a67c9956326b218fc307f8b9139af7 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 5 Dec 2018 17:29:59 -0800 Subject: [PATCH 21/22] Use ip route commands on Linux, as they show the subnet which may be required for deletion --- hack/jenkins/common.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index fe156a3a5c32..0f745d9d8041 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -162,17 +162,19 @@ if pgrep kubectl; then fi function cleanup_stale_routes() { - stale_routes=$(netstat -rn -f inet | awk '{ print $1 }' | grep 10.96.0.0 || true) - if [[ "${stale_routes}" != "" ]]; then - echo "WARNING: deleting stale tunnel routes: ${stale_routes}" - for route in ${stale_routes}; do - if [[ "$(uname)" == "Linux" ]]; then - sudo ip route delete "${route}" || true - else - sudo route -n delete "${route}" || true - fi - done + local show="netstat -rn -f inet" + local del="sudo route -n delete" + + if [[ "$(uname)" == "Linux" ]]; then + show="ip route show" + del="sudo ip route delete" fi + + local troutes=$($show | awk '{ print $1 }' | grep 10.96.0.0 || true) + for route in ${troutes}; do + echo "WARNING: deleting stale tunnel route: ${route}" + $del "${route}" || true + done } cleanup_stale_routes || true From 2847622892cc609988ee956e0173ffc448bf40ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Str=C3=B6mberg?= Date: Thu, 6 Dec 2018 09:05:33 -0800 Subject: [PATCH 22/22] Revert "Make k8s v1.12.3 the default" --- Makefile | 3 +-- pkg/minikube/constants/constants.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 43fc7b6bc416..290d625ab2d9 100755 --- a/Makefile +++ b/Makefile @@ -23,8 +23,7 @@ BUILDROOT_BRANCH ?= 2018.05 REGISTRY?=gcr.io/k8s-minikube HYPERKIT_BUILD_IMAGE ?= karalabe/xgo-1.10.x -# NOTE: "latest" as of 2018-12-04. kube-cross images aren't updated as often as Kubernetes -BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.11.1-1 +BUILD_IMAGE ?= k8s.gcr.io/kube-cross:v1.10.1-1 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image ISO_VERSION ?= v0.30.0 diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 026ff0398333..0b9966dc580e 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -22,13 +22,13 @@ import ( "path/filepath" "runtime" "strings" - "time" "github.com/blang/semver" "github.com/golang/glog" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" minikubeVersion "k8s.io/minikube/pkg/version" + "time" ) // APIServerPort is the port that the API server should listen on. @@ -125,7 +125,7 @@ const ( var DefaultIsoUrl = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetIsoPath(), minikubeVersion.GetIsoVersion()) var DefaultIsoShaUrl = DefaultIsoUrl + ShaSuffix -var DefaultKubernetesVersion = "v1.12.3" +var DefaultKubernetesVersion = "v1.10.0" var ConfigFilePath = MakeMiniPath("config") var ConfigFile = MakeMiniPath("config", "config.json")