Merge pull request #11150 from Luap99/v3.3-netcon

[v3.3] fix rootless port forwarding with network dis-/connect

docs/source/markdown/podman-network-connect.1.md

@@ -10,8 +10,6 @@ podman\-network\-connect - Connect a container to a network
 Connects a container to a network. A container can be connected to a network by name or by ID.
 Once connected, the container can communicate with other containers in the same network.
 
-This command is not available for rootless users.
-
 ## OPTIONS
 #### **--alias**
 Add network-scoped alias for the container.  If the network is using the `dnsname` CNI plugin, these aliases

docs/source/markdown/podman-network-disconnect.1.md

@@ -7,9 +7,10 @@ podman\-network\-disconnect - Disconnect a container from a network
 **podman network disconnect** [*options*] network container
 
 ## DESCRIPTION
-Disconnects a container from a network.
+Disconnects a container from a network. A container can be disconnected from a network by name or by ID.
+If all networks are disconnected from the container, it will behave like a container created with `--network=none`
+and it will longer have network connectivity until a network is connected again.
 
-This command is not available for rootless users.
 
 ## OPTIONS
 #### **--force**, **-f**

docs/source/markdown/podman-network-reload.1.md

@@ -13,8 +13,6 @@ Rootfull Podman relies on iptables rules in order to provide network connectivit
 this happens for example with `firewall-cmd --reload`, the container loses network connectivity. This command restores
 the network connectivity.
 
-This command is not available for rootless users since rootless containers are not affected by such connectivity problems.
-
 ## OPTIONS
 #### **--all**, **-a**
 

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/opencontainers/selinux v1.8.2
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/rootless-containers/rootlesskit v0.14.2
+	github.com/rootless-containers/rootlesskit v0.14.4
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -395,7 +395,7 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -547,7 +547,6 @@ github.com/insomniacslk/dhcp v0.0.0-20210120172423-cc9239ac6294/go.mod h1:TKl4jN
 github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee h1:PAXLXk1heNZ5yokbMBpVLZQxo43wCZxRwl00mX+dd44=
 github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/jamescun/tuntap v0.0.0-20190712092105-cb1fb277045c/go.mod h1:zzwpsgcYhzzIP5WyF8g9ivCv38cY9uAV9Gu0m3lThhE=
 github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
 github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -802,8 +801,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rootless-containers/rootlesskit v0.14.2 h1:jmsSyNyRG0QdWc3usppt5jEy5qOheeUsIINcymPrOFg=
-github.com/rootless-containers/rootlesskit v0.14.2/go.mod h1:nV3TpRISvwhZQSwo0nmQQnxjCxXr3mvrMi0oASLvzcg=
+github.com/rootless-containers/rootlesskit v0.14.4 h1:pqx9a+OC/6jjV7sIUKy3D1p6NLEC6WIMiJWAGsGMCUM=
+github.com/rootless-containers/rootlesskit v0.14.4/go.mod h1:Ai3detLzryb/4EkzXmNfh8aByUcBXp/qqkQusJs1SO8=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -829,6 +828,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=

libpod/networking_linux.go

@@ -1214,7 +1214,29 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro
 		}
 	}
 	c.state.NetworkStatus = tmpNetworkStatus
-	return c.save()
+	err = c.save()
+	if err != nil {
+		return err
+	}
+
+	// OCICNI will set the loopback adpter down on teardown so we should set it up again
+	err = c.state.NetNS.Do(func(_ ns.NetNS) error {
+		link, err := netlink.LinkByName("lo")
+		if err != nil {
+			return err
+		}
+		err = netlink.LinkSetUp(link)
+		return err
+	})
+	if err != nil {
+		logrus.Warnf("failed to set loopback adpter up in the container: %v", err)
+	}
+	// Reload ports when there are still connected networks, maybe we removed the network interface with the child ip.
+	// Reloading without connected networks does not make sense, so we can skip this step.
+	if rootless.IsRootless() && len(tmpNetworkStatus) > 0 {
+		return c.reloadRootlessRLKPortMapping()
+	}
+	return nil
 }
 
 // ConnectNetwork connects a container to a given network
@@ -1306,7 +1328,16 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e
 		networkStatus[index] = networkResults[0]
 		c.state.NetworkStatus = networkStatus
 	}
-	return c.save()
+	err = c.save()
+	if err != nil {
+		return err
+	}
+	// The first network needs a port reload to set the correct child ip for the rootlessport process.
+	// Adding a second network does not require a port reload because the child ip is still valid.
+	if rootless.IsRootless() && len(networks) == 0 {
+		return c.reloadRootlessRLKPortMapping()
+	}
+	return nil
 }
 
 // DisconnectContainerFromNetwork removes a container from its CNI network

libpod/networking_slirp4netns.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/containers/podman/v3/pkg/errorhandling"
+	"github.com/containers/podman/v3/pkg/rootless"
 	"github.com/containers/podman/v3/pkg/rootlessport"
 	"github.com/containers/podman/v3/pkg/servicereaper"
 	"github.com/pkg/errors"
@@ -466,29 +467,16 @@ func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath strin
 		}
 	}
 
-	slirp4netnsIP, err := GetSlirp4netnsIP(ctr.slirp4netnsSubnet)
-	if err != nil {
-		return errors.Wrapf(err, "failed to get slirp4ns ip")
-	}
-	childIP := slirp4netnsIP.String()
-outer:
-	for _, r := range ctr.state.NetworkStatus {
-		for _, i := range r.IPs {
-			ipv4 := i.Address.IP.To4()
-			if ipv4 != nil {
-				childIP = ipv4.String()
-				break outer
-			}
-		}
-	}
-
+	childIP := getRootlessPortChildIP(ctr)
 	cfg := rootlessport.Config{
-		Mappings:  ctr.config.PortMappings,
-		NetNSPath: netnsPath,
-		ExitFD:    3,
-		ReadyFD:   4,
-		TmpDir:    ctr.runtime.config.Engine.TmpDir,
-		ChildIP:   childIP,
+		Mappings:    ctr.config.PortMappings,
+		NetNSPath:   netnsPath,
+		ExitFD:      3,
+		ReadyFD:     4,
+		TmpDir:      ctr.runtime.config.Engine.TmpDir,
+		ChildIP:     childIP,
+		ContainerID: ctr.config.ID,
+		RootlessCNI: ctr.config.NetMode.IsBridge() && rootless.IsRootless(),
 	}
 	cfgJSON, err := json.Marshal(cfg)
 	if err != nil {
@@ -617,3 +605,62 @@ func (r *Runtime) setupRootlessPortMappingViaSlirp(ctr *Container, cmd *exec.Cmd
 	logrus.Debug("slirp4netns port-forwarding setup via add_hostfwd is ready")
 	return nil
 }
+
+func getRootlessPortChildIP(c *Container) string {
+	if c.config.NetMode.IsSlirp4netns() {
+		slirp4netnsIP, err := GetSlirp4netnsIP(c.slirp4netnsSubnet)
+		if err != nil {
+			return ""
+		}
+		return slirp4netnsIP.String()
+	}
+
+	for _, r := range c.state.NetworkStatus {
+		for _, i := range r.IPs {
+			ipv4 := i.Address.IP.To4()
+			if ipv4 != nil {
+				return ipv4.String()
+			}
+		}
+	}
+	return ""
+}
+
+// reloadRootlessRLKPortMapping will trigger a reload for the port mappings in the rootlessport process.
+// This should only be called by network connect/disconnect and only as rootless.
+func (c *Container) reloadRootlessRLKPortMapping() error {
+	childIP := getRootlessPortChildIP(c)
+	logrus.Debugf("reloading rootless ports for container %s, childIP is %s", c.config.ID, childIP)
+
+	var conn net.Conn
+	var err error
+	// try three times to connect to the socket, maybe it is not ready yet
+	for i := 0; i < 3; i++ {
+		conn, err = net.Dial("unix", filepath.Join(c.runtime.config.Engine.TmpDir, "rp", c.config.ID))
+		if err == nil {
+			break
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	if err != nil {
+		// This is not a hard error for backwards compatibility. A container started
+		// with an old version did not created the rootlessport socket.
+		logrus.Warnf("Could not reload rootless port mappings, port forwarding may no longer work correctly: %v", err)
+		return nil
+	}
+	defer conn.Close()
+	enc := json.NewEncoder(conn)
+	err = enc.Encode(childIP)
+	if err != nil {
+		return errors.Wrap(err, "port reloading failed")
+	}
+	b, err := ioutil.ReadAll(conn)
+	if err != nil {
+		return errors.Wrap(err, "port reloading failed")
+	}
+	data := string(b)
+	if data != "OK" {
+		return errors.Errorf("port reloading failed: %s", data)
+	}
+	return nil
+}

pkg/rootlessport/rootlessport_linux.go

@@ -17,9 +17,11 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containers/storage/pkg/reexec"
@@ -43,12 +45,14 @@ const (
 // Config needs to be provided to the process via stdin as a JSON string.
 // stdin needs to be closed after the message has been written.
 type Config struct {
-	Mappings  []ocicni.PortMapping
-	NetNSPath string
-	ExitFD    int
-	ReadyFD   int
-	TmpDir    string
-	ChildIP   string
+	Mappings    []ocicni.PortMapping
+	NetNSPath   string
+	ExitFD      int
+	ReadyFD     int
+	TmpDir      string
+	ChildIP     string
+	ContainerID string
+	RootlessCNI bool
 }
 
 func init() {
@@ -126,6 +130,12 @@ func parent() error {
 		}
 	}()
 
+	socketDir := filepath.Join(cfg.TmpDir, "rp")
+	err = os.MkdirAll(socketDir, 0700)
+	if err != nil {
+		return err
+	}
+
 	// create the parent driver
 	stateDir, err := ioutil.TempDir(cfg.TmpDir, "rootlessport")
 	if err != nil {
@@ -231,6 +241,16 @@ outer:
 		return err
 	}
 
+	// we only need to have a socket to reload ports when we run under rootless cni
+	if cfg.RootlessCNI {
+		socket, err := net.Listen("unix", filepath.Join(socketDir, cfg.ContainerID))
+		if err != nil {
+			return err
+		}
+		defer socket.Close()
+		go serve(socket, driver)
+	}
+
 	// write and close ReadyFD (convention is same as slirp4netns --ready-fd)
 	logrus.Info("ready")
 	if _, err := readyW.Write([]byte("1")); err != nil {
@@ -248,6 +268,53 @@ outer:
 	return nil
 }
 
+func serve(listener net.Listener, pm rkport.Manager) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			// we cannot log this error, stderr is already closed
+			continue
+		}
+		ctx := context.TODO()
+		err = handler(ctx, conn, pm)
+		if err != nil {
+			conn.Write([]byte(err.Error()))
+		} else {
+			conn.Write([]byte("OK"))
+		}
+		conn.Close()
+	}
+}
+
+func handler(ctx context.Context, conn io.Reader, pm rkport.Manager) error {
+	var childIP string
+	dec := json.NewDecoder(conn)
+	err := dec.Decode(&childIP)
+	if err != nil {
+		return errors.Wrap(err, "rootless port failed to decode ports")
+	}
+	portStatus, err := pm.ListPorts(ctx)
+	if err != nil {
+		return errors.Wrap(err, "rootless port failed to list ports")
+	}
+	for _, status := range portStatus {
+		err = pm.RemovePort(ctx, status.ID)
+		if err != nil {
+			return errors.Wrap(err, "rootless port failed to remove port")
+		}
+	}
+	// add the ports with the new child IP
+	for _, status := range portStatus {
+		// set the new child IP
+		status.Spec.ChildIP = childIP
+		_, err = pm.AddPort(ctx, status.Spec)
+		if err != nil {
+			return errors.Wrap(err, "rootless port failed to add port")
+		}
+	}
+	return nil
+}
+
 func exposePorts(pm rkport.Manager, portMappings []ocicni.PortMapping, childIP string) error {
 	ctx := context.TODO()
 	for _, i := range portMappings {