From e7907e3caaf85effdf97a353428e6a5b308a96da Mon Sep 17 00:00:00 2001
From: Predrag Rogic <prezha@users.noreply.github.com>
Date: Mon, 3 May 2021 01:28:46 +0100
Subject: [PATCH] fix kubelet "root container [kubepods] doesnt exist"

---
 pkg/minikube/bootstrapper/bsutil/kubelet.go | 10 ++++++++++
 pkg/minikube/node/start.go                  | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go
index 76043824afaa..9b99f1a88ad0 100644
--- a/pkg/minikube/bootstrapper/bsutil/kubelet.go
+++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go
@@ -23,10 +23,12 @@ import (
 	"path"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/util"
@@ -47,6 +49,14 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 	for k, v := range r.KubeletOptions() {
 		extraOpts[k] = v
 	}
+
+	// avoid "Failed to start ContainerManager failed to initialise top level QOS containers" error (ref: https://github.com/kubernetes/kubernetes/issues/43856)
+	// avoid "kubelet crashes with: root container [kubepods] doesn't exist" (ref: https://github.com/kubernetes/kubernetes/issues/95488)
+	if mc.Driver == oci.Docker && mc.KubernetesConfig.ContainerRuntime == constants.CRIO {
+		extraOpts["cgroups-per-qos"] = "false"
+		extraOpts["enforce-node-allocatable"] = ""
+	}
+
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
 
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index b35bd39ff417..e5ae1f56ff70 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/cni"
@@ -236,6 +237,15 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe
 		klog.Infof("trying to join worker node %q to cluster: %+v", starter.Node.Name, starter.Node)
 		if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil {
 			klog.Errorf("worker node failed to join cluster, will retry: %v", err)
+
+			// reset worker node to revert any changes made by previous kubeadm init/join
+			klog.Infof("resetting worker node %q before attempting to rejoin cluster...", starter.Node.Name)
+			if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil {
+				klog.Infof("kubeadm reset failed, continuing anyway: %v", err)
+			} else {
+				klog.Infof("successfully reset worker node %q", starter.Node.Name)
+			}
+
 			return err
 		}
 		return nil