diff --git a/README.md b/README.md index 3f002b371302..02308a05e2c2 100644 --- a/README.md +++ b/README.md @@ -23,20 +23,20 @@ minikube implements a local Kubernetes cluster on macOS, Linux, and Windows. min minikube runs the latest stable release of Kubernetes, with support for standard Kubernetes features like: -* [LoadBalancer](https://minikube.sigs.k8s.io/docs/tasks/loadbalancer/) - using `minikube tunnel` +* [LoadBalancer](https://minikube.sigs.k8s.io/docs/handbook/accessing/#loadbalancer-access) - using `minikube tunnel` * Multi-cluster - using `minikube start -p ` * NodePorts - using `minikube service` -* [Persistent Volumes](https://minikube.sigs.k8s.io/docs/reference/persistent_volumes/) +* [Persistent Volumes](https://minikube.sigs.k8s.io/docs/handbook/persistent_volumes/) * [Ingress](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/) -* [Dashboard](https://minikube.sigs.k8s.io/docs/tasks/dashboard/) - `minikube dashboard` -* [Container runtimes](https://minikube.sigs.k8s.io/docs/reference/runtimes/) - `start --container-runtime` -* [Configure apiserver and kubelet options](https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/) via command-line flags +* [Dashboard](https://minikube.sigs.k8s.io/docs/handbook/dashboard/) - `minikube dashboard` +* [Container runtimes](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) - `start --container-runtime` +* [Configure apiserver and kubelet options](https://minikube.sigs.k8s.io/docs/handbook/config/#modifying-kubernetes-defaults) via command-line flags As well as developer-friendly features: -* [Addons](https://minikube.sigs.k8s.io/docs/tasks/addons/) - a marketplace for developers to share configurations for running services on minikube +* [Addons](https://minikube.sigs.k8s.io/docs/handbook/deploying/#addons) - a marketplace for developers to share configurations for running services on minikube * [NVIDIA GPU support](https://minikube.sigs.k8s.io/docs/tutorials/nvidia_gpu/) - for machine learning -* [Filesystem mounts](https://minikube.sigs.k8s.io/docs/tasks/mount/) +* [Filesystem mounts](https://minikube.sigs.k8s.io/docs/handbook/mount/) **For more information, see the official [minikube website](https://minikube.sigs.k8s.io)** diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 22c16062d3d6..b25696ae3458 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -329,23 +329,24 @@ func profileDeletionErr(cname string, additionalInfo string) error { func uninstallKubernetes(api libmachine.API, cc config.ClusterConfig, n config.Node, bsName string) error { out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": cc.KubernetesConfig.KubernetesVersion, "bootstrapper_name": bsName}) - clusterBootstrapper, err := cluster.Bootstrapper(api, bsName, cc, n) + host, err := machine.LoadHost(api, driver.MachineName(cc, n)) if err != nil { - return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal} + return DeletionError{Err: fmt.Errorf("unable to load host: %v", err), Errtype: MissingCluster} } - host, err := machine.LoadHost(api, driver.MachineName(cc, n)) + r, err := machine.CommandRunner(host) if err != nil { - exit.WithError("Error getting host", err) + return DeletionError{Err: fmt.Errorf("unable to get command runner %v", err), Errtype: MissingCluster} } - r, err := machine.CommandRunner(host) + + clusterBootstrapper, err := cluster.Bootstrapper(api, bsName, cc, r) if err != nil { - exit.WithError("Failed to get command runner", err) + return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal} } cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r}) if err != nil { - exit.WithError("Failed runtime", err) + return DeletionError{Err: fmt.Errorf("unable to get runtime: %v", err), Errtype: Fatal} } // Unpause the cluster if necessary to avoid hung kubeadm diff --git a/cmd/minikube/cmd/logs.go b/cmd/minikube/cmd/logs.go index 109938dc34c4..7dfc2df24f0b 100644 --- a/cmd/minikube/cmd/logs.go +++ b/cmd/minikube/cmd/logs.go @@ -53,7 +53,7 @@ var logsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { co := mustload.Running(ClusterFlagValue()) - bs, err := cluster.Bootstrapper(co.API, viper.GetString(cmdcfg.Bootstrapper), *co.Config, *co.CP.Node) + bs, err := cluster.Bootstrapper(co.API, viper.GetString(cmdcfg.Bootstrapper), *co.Config, co.CP.Runner) if err != nil { exit.WithError("Error getting cluster bootstrapper", err) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 735032af120c..3be149ae2fa6 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -68,18 +68,8 @@ type Bootstrapper struct { } // NewBootstrapper creates a new kubeadm.Bootstrapper -// TODO(#6891): Remove node as an argument -func NewBootstrapper(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Bootstrapper, error) { - name := driver.MachineName(cc, n) - h, err := api.Load(name) - if err != nil { - return nil, errors.Wrap(err, "getting api client") - } - runner, err := machine.CommandRunner(h) - if err != nil { - return nil, errors.Wrap(err, "command runner") - } - return &Bootstrapper{c: runner, contextName: cc.Name, k8sClient: nil}, nil +func NewBootstrapper(api libmachine.API, cc config.ClusterConfig, r command.Runner) (*Bootstrapper, error) { + return &Bootstrapper{c: r, contextName: cc.Name, k8sClient: nil}, nil } // GetAPIServerStatus returns the api-server status @@ -111,8 +101,7 @@ func (k *Bootstrapper) LogCommands(cfg config.ClusterConfig, o bootstrapper.LogO dmesg.WriteString(fmt.Sprintf(" | tail -n %d", o.Lines)) } - describeNodes := fmt.Sprintf("sudo %s describe nodes --kubeconfig=%s", - path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), + describeNodes := fmt.Sprintf("sudo %s describe nodes --kubeconfig=%s", kubectlPath(cfg), path.Join(vmpath.GuestPersistentDir, "kubeconfig")) return map[string]string{ @@ -218,7 +207,7 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { go func() { // the overlay is required for containerd and cri-o runtime: see #7428 if driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker" { - if err := k.applyKicOverlay(cfg); err != nil { + if err := k.applyKICOverlay(cfg); err != nil { glog.Errorf("failed to apply kic overlay: %v", err) } } @@ -704,7 +693,7 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru } // Installs compatibility shims for non-systemd environments - kubeletPath := path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") + kubeletPath := path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubelet") shims, err := sm.GenerateInitShim("kubelet", kubeletPath, bsutil.KubeletSystemdConfFile) if err != nil { return errors.Wrap(err, "shim") @@ -764,21 +753,32 @@ func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { return sm.Start("kubelet") } -// applyKicOverlay applies the CNI plugin needed to make kic work -func (k *Bootstrapper) applyKicOverlay(cfg config.ClusterConfig) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, "sudo", - path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), "create", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")), - "-f", "-") +// kubectlPath returns the path to the kubelet +func kubectlPath(cfg config.ClusterConfig) string { + return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") +} +// applyKICOverlay applies the CNI plugin needed to make kic work +func (k *Bootstrapper) applyKICOverlay(cfg config.ClusterConfig) error { b := bytes.Buffer{} if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil { return err } - cmd.Stdin = bytes.NewReader(b.Bytes()) + ko := path.Join(vmpath.GuestEphemeralDir, fmt.Sprintf("kic_overlay.yaml")) + f := assets.NewMemoryAssetTarget(b.Bytes(), ko, "0644") + + if err := k.c.Copy(f); err != nil { + return errors.Wrapf(err, "copy") + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "sudo", kubectlPath(cfg), "apply", + fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")), + "-f", ko) + if rr, err := k.c.RunCmd(cmd); err != nil { return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) } @@ -807,8 +807,7 @@ func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { defer cancel() // example: // sudo /var/lib/minikube/binaries//kubectl label nodes minikube.k8s.io/version= minikube.k8s.io/commit=aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name=p1 minikube.k8s.io/updated_at=2020_02_20T12_05_35_0700 --all --overwrite --kubeconfig=/var/lib/minikube/kubeconfig - cmd := exec.CommandContext(ctx, "sudo", - path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), + cmd := exec.CommandContext(ctx, "sudo", kubectlPath(cfg), "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, "--all", "--overwrite", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) @@ -826,8 +825,7 @@ func (k *Bootstrapper) elevateKubeSystemPrivileges(cfg config.ClusterConfig) err defer cancel() rbacName := "minikube-rbac" // kubectl create clusterrolebinding minikube-rbac --clusterrole=cluster-admin --serviceaccount=kube-system:default - cmd := exec.CommandContext(ctx, "sudo", - path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), + cmd := exec.CommandContext(ctx, "sudo", kubectlPath(cfg), "create", "clusterrolebinding", rbacName, "--clusterrole=cluster-admin", "--serviceaccount=kube-system:default", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) rr, err := k.c.RunCmd(cmd) diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 878bd656625e..9e385686f2a8 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -26,6 +26,7 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" ) @@ -44,12 +45,12 @@ func init() { // Bootstrapper returns a new bootstrapper for the cluster // TODO(#6891): Remove node as an argument -func Bootstrapper(api libmachine.API, bootstrapperName string, cc config.ClusterConfig, n config.Node) (bootstrapper.Bootstrapper, error) { +func Bootstrapper(api libmachine.API, bootstrapperName string, cc config.ClusterConfig, r command.Runner) (bootstrapper.Bootstrapper, error) { var b bootstrapper.Bootstrapper var err error switch bootstrapperName { case bootstrapper.Kubeadm: - b, err = kubeadm.NewBootstrapper(api, cc, n) + b, err = kubeadm.NewBootstrapper(api, cc, r) if err != nil { return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper") } diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index 09b1a10c2449..9f59005f436d 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -17,15 +17,32 @@ limitations under the License. package cluster import ( + "time" + "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/sysinit" + "k8s.io/minikube/pkg/util/retry" ) -// Pause pauses a Kubernetes cluster +// Pause pauses a Kubernetes cluster, retrying if necessary func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { + var ids []string + tryPause := func() (err error) { + ids, err = pause(cr, r, namespaces) + return err + } + + if err := retry.Expo(tryPause, 250*time.Millisecond, 2*time.Second); err != nil { + return ids, err + } + return ids, nil +} + +// pause pauses a Kubernetes cluster +func pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { ids := []string{} // Disable the kubelet so it does not attempt to restart paused pods @@ -49,11 +66,24 @@ func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string } return ids, cr.PauseContainers(ids) - } -// Unpause unpauses a Kubernetes cluster +// Unpause unpauses a Kubernetes cluster, retrying if necessary func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { + var ids []string + tryUnpause := func() (err error) { + ids, err = unpause(cr, r, namespaces) + return err + } + + if err := retry.Expo(tryUnpause, 250*time.Millisecond, 2*time.Second); err != nil { + return ids, err + } + return ids, nil +} + +// unpause unpauses a Kubernetes cluster +func unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces}) if err != nil { return ids, errors.Wrap(err, "list paused") diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go index ce3cc58522bc..e8b79cb03190 100644 --- a/pkg/minikube/command/ssh_runner.go +++ b/pkg/minikube/command/ssh_runner.go @@ -25,12 +25,15 @@ import ( "sync" "time" + "github.com/docker/machine/libmachine/drivers" "github.com/golang/glog" "github.com/kballard/go-shellquote" "github.com/pkg/errors" "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/sshutil" + "k8s.io/minikube/pkg/util/retry" ) var ( @@ -41,13 +44,53 @@ var ( // // It implements the CommandRunner interface. type SSHRunner struct { + d drivers.Driver c *ssh.Client } // NewSSHRunner returns a new SSHRunner that will run commands // through the ssh.Client provided. -func NewSSHRunner(c *ssh.Client) *SSHRunner { - return &SSHRunner{c} +func NewSSHRunner(d drivers.Driver) *SSHRunner { + return &SSHRunner{d: d, c: nil} +} + +// client returns an ssh client (uses retry underneath) +func (s *SSHRunner) client() (*ssh.Client, error) { + if s.c != nil { + return s.c, nil + } + + c, err := sshutil.NewSSHClient(s.d) + if err != nil { + return nil, errors.Wrap(err, "new client") + } + s.c = c + return s.c, nil +} + +// session returns an ssh session, retrying if necessary +func (s *SSHRunner) session() (*ssh.Session, error) { + var sess *ssh.Session + getSession := func() (err error) { + client, err := s.client() + if err != nil { + return errors.Wrap(err, "new client") + } + + sess, err = client.NewSession() + if err != nil { + glog.Warningf("session error, resetting client: %v", err) + s.c = nil + return err + } + return nil + } + + if err := retry.Expo(getSession, 250*time.Millisecond, 2*time.Second); err != nil { + return nil, err + } + + return sess, nil } // Remove runs a command to delete a file on the remote. @@ -55,7 +98,7 @@ func (s *SSHRunner) Remove(f assets.CopyableFile) error { dst := path.Join(f.GetTargetDir(), f.GetTargetName()) glog.Infof("rm: %s", dst) - sess, err := s.c.NewSession() + sess, err := s.session() if err != nil { return errors.Wrap(err, "getting ssh session") } @@ -97,6 +140,10 @@ func teeSSH(s *ssh.Session, cmd string, outB io.Writer, errB io.Writer) error { // RunCmd implements the Command Runner interface to run a exec.Cmd object func (s *SSHRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { + if cmd.Stdin != nil { + return nil, fmt.Errorf("SSHRunner does not support stdin - you could be the first to add it") + } + rr := &RunResult{Args: cmd.Args} glog.Infof("Run: %v", rr.Command()) @@ -117,7 +164,7 @@ func (s *SSHRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { errb = io.MultiWriter(cmd.Stderr, &rr.Stderr) } - sess, err := s.c.NewSession() + sess, err := s.session() if err != nil { return rr, errors.Wrap(err, "NewSession") } @@ -170,10 +217,17 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error { glog.Warningf("0 byte asset: %+v", f) } - sess, err := s.c.NewSession() + sess, err := s.session() if err != nil { return errors.Wrap(err, "NewSession") } + defer func() { + if err := sess.Close(); err != nil { + if err != io.EOF { + glog.Errorf("session close: %v", err) + } + } + }() w, err := sess.StdinPipe() if err != nil { diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index 49f0a901beac..2d30fa9b817b 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -36,7 +36,7 @@ import ( "github.com/docker/machine/libmachine/host" "github.com/docker/machine/libmachine/mcnutils" "github.com/docker/machine/libmachine/persist" - "github.com/docker/machine/libmachine/ssh" + lmssh "github.com/docker/machine/libmachine/ssh" "github.com/docker/machine/libmachine/state" "github.com/docker/machine/libmachine/swarm" "github.com/docker/machine/libmachine/version" @@ -48,13 +48,12 @@ import ( "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/registry" - "k8s.io/minikube/pkg/minikube/sshutil" ) // NewRPCClient gets a new client. func NewRPCClient(storePath, certsDir string) libmachine.API { c := libmachine.NewClient(storePath, certsDir) - c.SSHClientType = ssh.Native + c.SSHClientType = lmssh.Native return c } @@ -154,19 +153,7 @@ func CommandRunner(h *host.Host) (command.Runner, error) { return command.NewExecRunner(), nil } - if driver.IsKIC(h.Driver.DriverName()) { - return command.NewKICRunner(h.Name, h.Driver.DriverName()), nil - } - return SSHRunner(h) -} - -// SSHRunner returns an SSH runner for the host -func SSHRunner(h *host.Host) (command.Runner, error) { - client, err := sshutil.NewSSHClient(h.Driver) - if err != nil { - return nil, errors.Wrap(err, "getting ssh client for bootstrapper") - } - return command.NewSSHRunner(client), nil + return command.NewSSHRunner(h.Driver), nil } // Create creates the host diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index e42fc9cf62ca..3ad987acdca2 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -32,19 +32,15 @@ import ( "github.com/juju/mutex" "github.com/pkg/errors" "github.com/spf13/viper" - "golang.org/x/crypto/ssh" "k8s.io/minikube/pkg/drivers/kic/oci" - "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/registry" - "k8s.io/minikube/pkg/minikube/sshutil" "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util/lock" - "k8s.io/minikube/pkg/util/retry" ) var ( @@ -198,7 +194,7 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { glog.Infof("creating required directories: %v", requiredDirectories) - r, err := commandRunner(h) + r, err := CommandRunner(h) if err != nil { return errors.Wrap(err, "command runner") } @@ -217,40 +213,6 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { return syncLocalAssets(r) } -// commandRunner returns best available command runner for this host -func commandRunner(h *host.Host) (command.Runner, error) { - d := h.Driver.DriverName() - glog.V(1).Infof("determining appropriate runner for %q", d) - if driver.IsMock(d) { - glog.Infof("returning FakeCommandRunner for %q driver", d) - return &command.FakeCommandRunner{}, nil - } - - if driver.BareMetal(h.Driver.DriverName()) { - glog.Infof("returning ExecRunner for %q driver", d) - return command.NewExecRunner(), nil - } - if driver.IsKIC(d) { - glog.Infof("Returning KICRunner for %q driver", d) - return command.NewKICRunner(h.Name, d), nil - } - - glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d) - - // Retry in order to survive an ssh restart, which sometimes happens due to provisioning - var sc *ssh.Client - getSSH := func() (err error) { - sc, err = sshutil.NewSSHClient(h.Driver) - return err - } - - if err := retry.Expo(getSSH, 250*time.Millisecond, 2*time.Second); err != nil { - return nil, err - } - - return command.NewSSHRunner(sc), nil -} - // acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup) func acquireMachinesLock(name string) (mutex.Releaser, error) { spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines")) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index b469ba6e168f..8949f3a6028f 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -89,18 +89,6 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) showVersionInfo(starter.Node.KubernetesVersion, cr) - // ssh should be set up by now - // switch to using ssh runner since it is faster - if driver.IsKIC(starter.Cfg.Driver) { - sshRunner, err := machine.SSHRunner(starter.Host) - if err != nil { - glog.Infof("error getting ssh runner: %v", err) - } else { - glog.Infof("Using ssh runner for kic...") - starter.Runner = sshRunner - } - } - var bs bootstrapper.Bootstrapper var kcs *kubeconfig.Settings if apiServer { @@ -111,7 +99,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } // setup kubeadm (must come after setupKubeconfig) - bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node) + bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner) err = bs.StartCluster(*starter.Cfg) if err != nil { @@ -124,7 +112,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { return nil, errors.Wrap(err, "Failed to update kubeconfig file.") } } else { - bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, *starter.Node) + bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner) if err != nil { return nil, errors.Wrap(err, "Failed to get bootstrapper") } @@ -168,11 +156,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { return nil, errors.Wrap(err, "Updating node") } - cp, err := config.PrimaryControlPlane(starter.Cfg) - if err != nil { - return nil, errors.Wrap(err, "Getting primary control plane") - } - cpBs, err := cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, cp) + cpBs, err := cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner) if err != nil { return nil, errors.Wrap(err, "Getting bootstrapper") } @@ -268,8 +252,8 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k } // setupKubeAdm adds any requested files into the VM before Kubernetes is started -func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, n config.Node) bootstrapper.Bootstrapper { - bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, n) +func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, n config.Node, r command.Runner) bootstrapper.Bootstrapper { + bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, r) if err != nil { exit.WithError("Failed to get bootstrapper", err) } diff --git a/pkg/minikube/sshutil/sshutil.go b/pkg/minikube/sshutil/sshutil.go index 3411f7e400d1..a8a91050c305 100644 --- a/pkg/minikube/sshutil/sshutil.go +++ b/pkg/minikube/sshutil/sshutil.go @@ -19,11 +19,14 @@ package sshutil import ( "net" "strconv" + "time" "github.com/docker/machine/libmachine/drivers" machinessh "github.com/docker/machine/libmachine/ssh" + "github.com/golang/glog" "github.com/pkg/errors" "golang.org/x/crypto/ssh" + "k8s.io/minikube/pkg/util/retry" ) // NewSSHClient returns an SSH client object for running commands. @@ -37,15 +40,27 @@ func NewSSHClient(d drivers.Driver) (*ssh.Client, error) { if h.SSHKeyPath != "" { auth.Keys = []string{h.SSHKeyPath} } + + glog.Infof("new ssh client: %+v", h) + config, err := machinessh.NewNativeConfig(h.Username, auth) if err != nil { return nil, errors.Wrapf(err, "Error creating new native config from ssh using: %s, %s", h.Username, auth) } - client, err := ssh.Dial("tcp", net.JoinHostPort(h.IP, strconv.Itoa(h.Port)), &config) - if err != nil { - return nil, errors.Wrap(err, "Error dialing tcp via ssh client") + var client *ssh.Client + getSSH := func() (err error) { + client, err = ssh.Dial("tcp", net.JoinHostPort(h.IP, strconv.Itoa(h.Port)), &config) + if err != nil { + glog.Warningf("dial failure (will retry): %v", err) + } + return err } + + if err := retry.Expo(getSSH, 250*time.Millisecond, 2*time.Second); err != nil { + return nil, err + } + return client, nil } diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index 7b2e9e653984..c3c2287e6770 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -39,7 +39,6 @@ import ( "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/sshutil" ) // generic interface for minikube provisioner @@ -165,11 +164,7 @@ func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error { authOptions.ServerKeyPath: authOptions.ServerKeyRemotePath, } - sshClient, err := sshutil.NewSSHClient(driver) - if err != nil { - return errors.Wrap(err, "provisioning: error getting ssh client") - } - sshRunner := command.NewSSHRunner(sshClient) + sshRunner := command.NewSSHRunner(driver) dirs := []string{} for _, dst := range remoteCerts { @@ -177,7 +172,7 @@ func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error { } args := append([]string{"mkdir", "-p"}, dirs...) - if _, err = sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil { + if _, err := sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil { return err } diff --git a/site/content/en/docs/_index.md b/site/content/en/docs/_index.md index 1a552191eb93..9cb97620e9c8 100755 --- a/site/content/en/docs/_index.md +++ b/site/content/en/docs/_index.md @@ -17,9 +17,9 @@ minikube quickly sets up a local Kubernetes cluster on macOS, Linux, and Windows * Cross-platform (Linux, macOS, Windows) * Deploy as a VM, a container, or on bare-metal * Multiple container runtimes (CRI-O, containerd, docker) -* Docker API endpoint for blazing fast [image pushes](https://minikube.sigs.k8s.io/docs/handbook/pushing/#pushing-directly-to-the-in-cluster-docker-daemon) -* Advanced features such as [LoadBalancer](https://minikube.sigs.k8s.io/Handbook/loadbalancer/), filesystem mounts, and FeatureGates -* [Addons](https://minikube.sigs.k8s.io/Handbook/addons/) for easily installed Kubernetes applications +* Docker API endpoint for blazing fast [image pushes]({{< ref "/docs/handbook/pushing.md#pushing-directly-to-the-in-cluster-docker-daemon" >}}) +* Advanced features such as [LoadBalancer]({{< ref "/docs/handbook/accessing.md#loadbalancer-access" >}}), filesystem mounts, and FeatureGates +* [Addons]({{< ref "/docs/handbook/deploying.md#addons" >}}) for easily installed Kubernetes applications ## Survey diff --git a/site/content/en/docs/drivers/includes/registry_addon_mac_windows.inc b/site/content/en/docs/drivers/includes/registry_addon_mac_windows.inc new file mode 100644 index 000000000000..c838c44e6664 --- /dev/null +++ b/site/content/en/docs/drivers/includes/registry_addon_mac_windows.inc @@ -0,0 +1,28 @@ +## Requirements + +* Run on Mac OS or Windows +* Registry addon enabled +* Run Minikube on KIC driver. + +## Start Minikube enabling registry addon + +Open a shell console, and run the following command: + +```bash +minikube start --addons=registry --vm-driver={{docker | podman}} +``` + +## Usage Registry + +Build docker image and tag it appropriately: + +```shell +docker build --tag $(minikube ip):5000/test-img . +``` +Push docker image to minikube registry: + +```shell +docker push $(minikube ip):5000/test-img +``` + +* For more information, documentation ([Pushing to an in-cluster using Registry addon](https://minikube.sigs.k8s.io/docs/handbook/pushing/#4-pushing-to-an-in-cluster-using-registry-addon)) diff --git a/site/content/en/docs/faq/_index.md b/site/content/en/docs/faq/_index.md index 321bff9bae5a..c077acab96c8 100644 --- a/site/content/en/docs/faq/_index.md +++ b/site/content/en/docs/faq/_index.md @@ -14,6 +14,6 @@ description: > The easiest approach is to use the `docker` driver, as the backend service always runs as `root`. -`none` users may want to try `CHANGE_MINIKUBE_NONE_USER=true`, where kubectl and such will still work: [see environment variables](https://minikube.sigs.k8s.io/reference/environment_variables/) +`none` users may want to try `CHANGE_MINIKUBE_NONE_USER=true`, where kubectl and such will still work: [see environment variables]({{< ref "/docs/handbook/config.md#environment-variables" >}}) Alternatively, configure `sudo` to never prompt for the commands issued by minikube. diff --git a/site/content/en/docs/handbook/accessing.md b/site/content/en/docs/handbook/accessing.md index 3b79ecfd2bb9..9b0aedae17b1 100644 --- a/site/content/en/docs/handbook/accessing.md +++ b/site/content/en/docs/handbook/accessing.md @@ -5,6 +5,7 @@ description: > How to access applications running within minikube aliases: - /docs/tasks/loadbalancer + - /Handbook/loadbalancer/ - /docs/tasks/nodeport --- diff --git a/site/content/en/docs/handbook/config.md b/site/content/en/docs/handbook/config.md index 9479c3c6914f..3578b27ab2f9 100644 --- a/site/content/en/docs/handbook/config.md +++ b/site/content/en/docs/handbook/config.md @@ -3,6 +3,10 @@ title: "Configuration" weight: 4 description: > Configuring your cluster +aliases: + - /docs/reference/environment_variables/ + - /docs/reference/configuration/kubernetes/ + - /docs/reference/runtimes --- ## Basic Configuration diff --git a/site/content/en/docs/handbook/deploying.md b/site/content/en/docs/handbook/deploying.md index 209d12b1a976..0c727afb0aac 100644 --- a/site/content/en/docs/handbook/deploying.md +++ b/site/content/en/docs/handbook/deploying.md @@ -5,6 +5,7 @@ description: > How to deploy an application to minikube aliases: - /docs/tasks/addons + - /Handbook/addons --- ## kubectl diff --git a/site/content/en/docs/handbook/filesync.md b/site/content/en/docs/handbook/filesync.md index bfa1551349ed..9b83831f484e 100644 --- a/site/content/en/docs/handbook/filesync.md +++ b/site/content/en/docs/handbook/filesync.md @@ -5,6 +5,7 @@ description: > How to sync files into minikube aliases: - /docs/tasks/sync/ + - /Handbook/sync/ --- ## Built-in sync diff --git a/site/content/en/docs/handbook/persistent_volumes.md b/site/content/en/docs/handbook/persistent_volumes.md index 0da5465b7b17..bb05a7944b1e 100644 --- a/site/content/en/docs/handbook/persistent_volumes.md +++ b/site/content/en/docs/handbook/persistent_volumes.md @@ -5,6 +5,8 @@ weight: 10 date: 2019-08-01 description: > About persistent volumes (hostPath) +aliases: + - /docs/reference/persistent_volumes --- minikube supports [PersistentVolumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) of type `hostPath` out of the box. These PersistentVolumes are mapped to a directory inside the running minikube instance (usually a VM, unless you use `--driver=none`, `--driver=docker`, or `--driver=podman`). For more information on how this works, read the Dynamic Provisioning section below. diff --git a/site/content/en/docs/tutorials/audit-policy.md b/site/content/en/docs/tutorials/audit-policy.md index 973986f0bb6b..74e87738e603 100644 --- a/site/content/en/docs/tutorials/audit-policy.md +++ b/site/content/en/docs/tutorials/audit-policy.md @@ -34,6 +34,6 @@ minikube start \ kubectl logs kube-apiserver-minikube -n kube-system | grep audit.k8s.io/v1 ``` -The [Audit Policy](https://kubernetes.io/docs/Handbook/debug-application-cluster/audit/#audit-policy) used in this tutorial is very minimal and quite verbose. As a next step you might want to finetune the `audit-policy.yaml` file. To get the changes applied you need to stop and start minikube. Restarting minikube triggers the [file sync mechanism](https://minikube.sigs.k8s.io/Handbook/sync/) that copies the yaml file onto the minikube node and causes the API server to read the changed policy file. +The [Audit Policy](https://kubernetes.io/docs/Handbook/debug-application-cluster/audit/#audit-policy) used in this tutorial is very minimal and quite verbose. As a next step you might want to finetune the `audit-policy.yaml` file. To get the changes applied you need to stop and start minikube. Restarting minikube triggers the [file sync mechanism]({{< ref "/docs/handbook/filesync.md" >}}) that copies the yaml file onto the minikube node and causes the API server to read the changed policy file. -Note: Currently there is no dedicated directory to store the `audit-policy.yaml` file in `~/.minikube/`. Using the `~/.minikube/files/etc/ssl/certs` directory is a workaround! This workaround works like this: By putting the file into a sub-directory of `~/.minikube/files/`, the [file sync mechanism](https://minikube.sigs.k8s.io/Handbook/sync/) gets triggered and copies the `audit-policy.yaml` file from the host onto the minikube node. When the API server container gets started by `kubeadm` I'll mount the `/etc/ssl/certs` directory from the minikube node into the container. This is the reason why the `audit-policy.yaml` file has to be stored in the ssl certs directory: It's one of the directories that get mounted from the minikube node into the container. +Note: Currently there is no dedicated directory to store the `audit-policy.yaml` file in `~/.minikube/`. Using the `~/.minikube/files/etc/ssl/certs` directory is a workaround! This workaround works like this: By putting the file into a sub-directory of `~/.minikube/files/`, the [file sync mechanism]({{< ref "/docs/handbook/filesync.md" >}}) gets triggered and copies the `audit-policy.yaml` file from the host onto the minikube node. When the API server container gets started by `kubeadm` I'll mount the `/etc/ssl/certs` directory from the minikube node into the container. This is the reason why the `audit-policy.yaml` file has to be stored in the ssl certs directory: It's one of the directories that get mounted from the minikube node into the container. diff --git a/site/content/en/docs/tutorials/openid_connect_auth.md b/site/content/en/docs/tutorials/openid_connect_auth.md index 769155c296fe..fefecd815e4d 100644 --- a/site/content/en/docs/tutorials/openid_connect_auth.md +++ b/site/content/en/docs/tutorials/openid_connect_auth.md @@ -13,7 +13,7 @@ Read more about OpenID Connect Authentication for Kubernetes here: }}) for more details. The following example configures your Minikube cluster to support RBAC and OIDC: