Merge pull request #11258 from prezha/fix-docker-crio

fix kubelet "root container [kubepods] doesnt exist"
kubernetes · May 5, 2021 · 6bca921 · 6bca921
2 parents b850829 + e7907e3
commit 6bca921
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -23,10 +23,12 @@ import (
 	"path"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/util"
@@ -47,6 +49,14 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 	for k, v := range r.KubeletOptions() {
 		extraOpts[k] = v
 	}
+
+	// avoid "Failed to start ContainerManager failed to initialise top level QOS containers" error (ref: https://github.com/kubernetes/kubernetes/issues/43856)
+	// avoid "kubelet crashes with: root container [kubepods] doesn't exist" (ref: https://github.com/kubernetes/kubernetes/issues/95488)
+	if mc.Driver == oci.Docker && mc.KubernetesConfig.ContainerRuntime == constants.CRIO {
+		extraOpts["cgroups-per-qos"] = "false"
+		extraOpts["enforce-node-allocatable"] = ""
+	}
+
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 

diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/cni"
@@ -236,6 +237,15 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe
 		klog.Infof("trying to join worker node %q to cluster: %+v", starter.Node.Name, starter.Node)
 		if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil {
 			klog.Errorf("worker node failed to join cluster, will retry: %v", err)
+
+			// reset worker node to revert any changes made by previous kubeadm init/join
+			klog.Infof("resetting worker node %q before attempting to rejoin cluster...", starter.Node.Name)
+			if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil {
+				klog.Infof("kubeadm reset failed, continuing anyway: %v", err)
+			} else {
+				klog.Infof("successfully reset worker node %q", starter.Node.Name)
+			}
+
 			return err
 		}
 		return nil