From 5195dd8eaeee453bb004e7689f7acf36285c8bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 16 Nov 2020 17:40:57 +0100 Subject: [PATCH] Add private network implementation for podman Most of it the same as docker, except for the options. i.e. libnetwork "bridge" plugin vs. cni "bridge" plugin --- cmd/minikube/cmd/delete.go | 2 +- pkg/drivers/kic/kic.go | 2 +- pkg/drivers/kic/oci/errors.go | 5 +- pkg/drivers/kic/oci/network.go | 19 ++++-- pkg/drivers/kic/oci/network_create.go | 94 ++++++++++++++------------- pkg/drivers/kic/oci/oci.go | 11 ++-- 6 files changed, 72 insertions(+), 61 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 9b45c582400d..8ce08f8d8582 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -260,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) { klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.DeleteKICNetworks() + errs = oci.DeleteKICNetworks(bin) if errs != nil { klog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs) } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 10c5aebdcc5d..d0bcf021f4ce 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -311,7 +311,7 @@ func (d *Driver) Remove() error { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } - if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil { + if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) } return nil diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 6089d88cb7d7..ce4aa00be45d 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -85,14 +85,15 @@ func LogContainerDebug(ociBin string, name string) string { } else { klog.Infof("postmortem docker info: %+v", di) } - logDockerNetworkInspect(name) + logDockerNetworkInspect(ociBin, name) } else { pi, err := podmanSystemInfo() if err != nil { - klog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err) + klog.Warningf("couldn't get postmortem podman info: %v", err) } else { klog.Infof("postmortem podman info: %+v", pi) } + logDockerNetworkInspect(ociBin, name) } if rr.Stdout.Len() == 0 { diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 9ca08fda28d2..a0bd87962bff 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -35,7 +35,7 @@ import ( func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - info, err := dockerNetworkInspect(clusterName) + info, err := dockerNetworkInspect(ociBin, clusterName) if err != nil { if errors.Is(err, ErrNetworkNotFound) { klog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube 0 { + if ociBin == Docker { + // options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options + args = append(args, "-o") + args = append(args, "--ip-masq") args = append(args, "-o") - args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + args = append(args, "--icc") + + // adding MTU option because #9528 + if mtu > 0 { + args = append(args, "-o") + args = append(args, fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu)) + } + + args = append(args, fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true")) } + args = append(args, name) - rr, err := runCmd(exec.Command(Docker, args...)) + rr, err := runCmd(exec.Command(ociBin, args...)) if err != nil { // Pool overlaps with other one on this address space if strings.Contains(rr.Output(), "Pool overlaps") { @@ -136,12 +138,16 @@ type netInfo struct { } // if exists returns subnet, gateway and mtu -func dockerNetworkInspect(name string) (netInfo, error) { +func dockerNetworkInspect(ociBin string, name string) (netInfo, error) { var info = netInfo{name: name} - cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}},{{(index .Options "com.docker.network.driver.mtu")}}`) + format := `{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}` + if ociBin == Docker { + format += `,{{(index .Options "com.docker.network.driver.mtu")}}` + } + cmd := exec.Command(ociBin, "network", "inspect", name, "--format", format) rr, err := runCmd(cmd) if err != nil { - logDockerNetworkInspect(name) + logDockerNetworkInspect(ociBin, name) if strings.Contains(rr.Output(), "No such network") { return info, ErrNetworkNotFound @@ -157,11 +163,13 @@ func dockerNetworkInspect(name string) (netInfo, error) { if len(vals) > 0 { info.gateway = net.ParseIP(vals[1]) - mtu, err := strconv.Atoi(vals[2]) - if err != nil { - klog.Warningf("couldn't parse mtu for docker network %q: %v", name, err) - } else { - info.mtu = mtu + if ociBin == Docker { + mtu, err := strconv.Atoi(vals[2]) + if err != nil { + klog.Warningf("couldn't parse mtu for docker network %q: %v", name, err) + } else { + info.mtu = mtu + } } } @@ -173,8 +181,8 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, nil } -func logDockerNetworkInspect(name string) { - cmd := exec.Command(Docker, "network", "inspect", name) +func logDockerNetworkInspect(ociBin string, name string) { + cmd := exec.Command(ociBin, "network", "inspect", name) klog.Infof("running %v to gather additional debugging logs...", cmd.Args) rr, err := runCmd(cmd) if err != nil { @@ -184,11 +192,11 @@ func logDockerNetworkInspect(name string) { } // RemoveNetwork removes a network -func RemoveNetwork(name string) error { - if !networkExists(name) { +func RemoveNetwork(ociBin string, name string) error { + if !networkExists(ociBin, name) { return nil } - rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + rr, err := runCmd(exec.Command(ociBin, "network", "rm", name)) if err != nil { if strings.Contains(rr.Output(), "No such network") { return ErrNetworkNotFound @@ -202,8 +210,8 @@ func RemoveNetwork(name string) error { return err } -func networkExists(name string) bool { - _, err := dockerNetworkInspect(name) +func networkExists(ociBin string, name string) bool { + _, err := dockerNetworkInspect(ociBin, name) if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error klog.Warningf("Error inspecting docker network %s: %v", name, err) } @@ -212,12 +220,8 @@ func networkExists(name string) bool { // networkNamesByLabel returns all network names created by a label func networkNamesByLabel(ociBin string, label string) ([]string, error) { - if ociBin != Docker { - return nil, fmt.Errorf("%s not supported", ociBin) - } - // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}' - rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + rr, err := runCmd(exec.Command(ociBin, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) if err != nil { return nil, err } @@ -231,14 +235,14 @@ func networkNamesByLabel(ociBin string, label string) ([]string, error) { } // DeleteKICNetworks deletes all networks created by kic -func DeleteKICNetworks() []error { +func DeleteKICNetworks(ociBin string) []error { var errs []error - ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true") + ns, err := networkNamesByLabel(ociBin, CreatedByLabelKey) if err != nil { return []error{errors.Wrap(err, "list all volume")} } for _, n := range ns { - err := RemoveNetwork(n) + err := RemoveNetwork(ociBin, n) if err != nil { errs = append(errs, err) } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 96783e58c2d3..d544c61d7822 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -146,6 +146,12 @@ func CreateContainerNode(p CreateParams) error { // label th enode wuth the node ID "--label", p.NodeLabel, } + // to provide a static IP + if p.Network != "" && p.IP != "" { + runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) + } + memcgSwap := true if runtime.GOOS == "linux" { if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { @@ -170,11 +176,6 @@ func CreateContainerNode(p CreateParams) error { virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { - // to provide a static IP for docker - if p.Network != "" && p.IP != "" { - runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) - } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")