diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 9b45c582400d..8ce08f8d8582 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) { klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.DeleteKICNetworks() + errs = oci.DeleteKICNetworks(bin) if errs != nil { klog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs) } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 4c47a478cf6f..156079675dc1 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -320,7 +320,7 @@ func (d *Driver) Remove() error { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } - if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil { + if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) } return nil diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 6089d88cb7d7..ce4aa00be45d 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -85,14 +85,15 @@ func LogContainerDebug(ociBin string, name string) string { } else { klog.Infof("postmortem docker info: %+v", di) } - logDockerNetworkInspect(name) + logDockerNetworkInspect(ociBin, name) } else { pi, err := podmanSystemInfo() if err != nil { - klog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err) + klog.Warningf("couldn't get postmortem podman info: %v", err) } else { klog.Infof("postmortem podman info: %+v", pi) } + logDockerNetworkInspect(ociBin, name) } if rr.Stdout.Len() == 0 { diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index a3c7edb94f6e..de2b742adf2c 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -35,7 +35,7 @@ import ( func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - info, err := dockerNetworkInspect(clusterName) + info, err := containerNetworkInspect(ociBin, clusterName) if err != nil { if errors.Is(err, ErrNetworkNotFound) { klog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube 0 { + if ociBin == Docker { + // options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options + args = append(args, "-o") + args = append(args, "--ip-masq") args = append(args, "-o") - args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + args = append(args, "--icc") + + // adding MTU option because #9528 + if mtu > 0 { + args = append(args, "-o") + args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + } + + args = append(args, fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true")) } + args = append(args, name) - rr, err := runCmd(exec.Command(Docker, args...)) + rr, err := runCmd(exec.Command(ociBin, args...)) if err != nil { // Pool overlaps with other one on this address space if strings.Contains(rr.Output(), "Pool overlaps") { @@ -135,6 +137,16 @@ type netInfo struct { mtu int } +func containerNetworkInspect(ociBin string, name string) (netInfo, error) { + if ociBin == Docker { + return dockerNetworkInspect(name) + } + if ociBin == Podman { + return podmanNetworkInspect(name) + } + return netInfo{}, fmt.Errorf("%s unknown", ociBin) +} + // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo type networkInspect struct { Name string @@ -153,7 +165,7 @@ func dockerNetworkInspect(name string) (netInfo, error) { cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{(index .Options "com.docker.network.driver.mtu")}},{{$first := true}} "ContainerIPs": [{{range $k,$v := .Containers }}{{if $first}}{{$first = false}}{{else}}, {{end}}"{{$v.IPv4Address}}"{{end}}]}`) rr, err := runCmd(cmd) if err != nil { - logDockerNetworkInspect(name) + logDockerNetworkInspect(Docker, name) if strings.Contains(rr.Output(), "No such network") { return info, ErrNetworkNotFound @@ -177,8 +189,39 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, nil } -func logDockerNetworkInspect(name string) { - cmd := exec.Command(Docker, "network", "inspect", name) +func podmanNetworkInspect(name string) (netInfo, error) { + var info = netInfo{name: name} + cmd := exec.Command(Podman, "network", "inspect", name, "--format", `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}`) + rr, err := runCmd(cmd) + if err != nil { + logDockerNetworkInspect(Podman, name) + if strings.Contains(rr.Output(), "No such network") { + + return info, ErrNetworkNotFound + } + return info, err + } + + // results looks like 172.17.0.0/16,172.17.0.1,1500 + vals := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",") + if len(vals) == 0 { + return info, fmt.Errorf("empty list network inspect: %q", rr.Output()) + } + + if len(vals) > 0 { + info.gateway = net.ParseIP(vals[1]) + } + + _, info.subnet, err = net.ParseCIDR(vals[0]) + if err != nil { + return info, errors.Wrapf(err, "parse subnet for %s", name) + } + + return info, nil +} + +func logDockerNetworkInspect(ociBin string, name string) { + cmd := exec.Command(ociBin, "network", "inspect", name) klog.Infof("running %v to gather additional debugging logs...", cmd.Args) rr, err := runCmd(cmd) if err != nil { @@ -188,11 +231,11 @@ func logDockerNetworkInspect(name string) { } // RemoveNetwork removes a network -func RemoveNetwork(name string) error { - if !networkExists(name) { +func RemoveNetwork(ociBin string, name string) error { + if !networkExists(ociBin, name) { return nil } - rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + rr, err := runCmd(exec.Command(ociBin, "network", "rm", name)) if err != nil { if strings.Contains(rr.Output(), "No such network") { return ErrNetworkNotFound @@ -206,8 +249,8 @@ func RemoveNetwork(name string) error { return err } -func networkExists(name string) bool { - _, err := dockerNetworkInspect(name) +func networkExists(ociBin string, name string) bool { + _, err := containerNetworkInspect(ociBin, name) if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error klog.Warningf("Error inspecting docker network %s: %v", name, err) } @@ -216,12 +259,8 @@ func networkExists(name string) bool { // networkNamesByLabel returns all network names created by a label func networkNamesByLabel(ociBin string, label string) ([]string, error) { - if ociBin != Docker { - return nil, fmt.Errorf("%s not supported", ociBin) - } - // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}' - rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + rr, err := runCmd(exec.Command(ociBin, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) if err != nil { return nil, err } @@ -235,14 +274,14 @@ func networkNamesByLabel(ociBin string, label string) ([]string, error) { } // DeleteKICNetworks deletes all networks created by kic -func DeleteKICNetworks() []error { +func DeleteKICNetworks(ociBin string) []error { var errs []error - ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true") + ns, err := networkNamesByLabel(ociBin, CreatedByLabelKey) if err != nil { return []error{errors.Wrap(err, "list all volume")} } for _, n := range ns { - err := RemoveNetwork(n) + err := RemoveNetwork(ociBin, n) if err != nil { errs = append(errs, err) } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 087d6224f4b6..63c45fc2833d 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -146,6 +146,12 @@ func CreateContainerNode(p CreateParams) error { // label th enode wuth the node ID "--label", p.NodeLabel, } + // to provide a static IP + if p.Network != "" && p.IP != "" { + runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) + } + memcgSwap := true if runtime.GOOS == "linux" { if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { @@ -170,11 +176,6 @@ func CreateContainerNode(p CreateParams) error { virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { - // to provide a static IP for docker - if p.Network != "" && p.IP != "" { - runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) - } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")