platform/qemu: Enable forwarding multiple ports

One of the limitations of unprivileged qemu is there is no
machine to machine networking.  Host port forwarding allows the
host machine to communicate to the guest machine through the local
host address. It also allows the guest machine to communicate
with the host machine through the gateway address. With two
guest machines, we can bridge the two guests through the host
machine with port forwarding. For example, Host A can run a
webserver on port 8080 and Host B can access Host A's webserver
through its gateway (host machine) address http://10.0.2.2:8080.

This is useful for creating test services (such as tang) for use
by the test machine.

mantle/cmd/kola/qemuexec.go

@@ -216,7 +216,10 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 		builder.Processors = -1
 	}
 	if usernet {
-		builder.EnableUsermodeNetworking(22)
+		h := []platform.HostForwardPort{
+			{Service: "ssh", HostPort: 0, GuestPort: 22},
+		}
+		builder.EnableUsermodeNetworking(h)
 	}
 	builder.InheritConsole = true
 	builder.Append(args...)

mantle/platform/machine/unprivqemu/cluster.go

@@ -123,7 +123,15 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo
 			return nil, errors.Wrapf(err, "adding additional disk")
 		}
 	}
-	builder.EnableUsermodeNetworking(22)
+
+	if len(options.HostForwardPorts) > 0 {
+		builder.EnableUsermodeNetworking(options.HostForwardPorts)
+	} else {
+		h := []platform.HostForwardPort{
+			{Service: "ssh", HostPort: 0, GuestPort: 22},
+		}
+		builder.EnableUsermodeNetworking(h)
+	}
 
 	inst, err := builder.Exec()
 	if err != nil {

mantle/platform/qemu.go

@@ -20,10 +20,9 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"os"
 	"path/filepath"
-	"regexp"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -33,16 +32,22 @@ import (
 	v3types "github.com/coreos/ignition/v2/config/v3_0/types"
 	"github.com/coreos/mantle/system"
 	"github.com/coreos/mantle/system/exec"
-	"github.com/coreos/mantle/util"
 	"github.com/pkg/errors"
 )
 
 var (
 	ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs")
 )
 
+type HostForwardPort struct {
+	Service   string
+	HostPort  int
+	GuestPort int
+}
+
 type MachineOptions struct {
-	AdditionalDisks []Disk
+	AdditionalDisks  []Disk
+	HostForwardPorts []HostForwardPort
 }
 
 type Disk struct {
@@ -61,11 +66,12 @@ type Disk struct {
 }
 
 type QemuInstance struct {
-	qemu       exec.Cmd
-	tmpConfig  string
-	tempdir    string
-	swtpm      exec.Cmd
-	nbdServers []exec.Cmd
+	qemu               exec.Cmd
+	tmpConfig          string
+	tempdir            string
+	swtpm              exec.Cmd
+	nbdServers         []exec.Cmd
+	hostForwardedPorts []HostForwardPort
 
 	journalPipe *os.File
 }
@@ -74,59 +80,11 @@ func (inst *QemuInstance) Pid() int {
 	return inst.qemu.Pid()
 }
 
-// parse /proc/net/tcp to determine the port selected by QEMU
+// Get the IP address with the forwarded port
 func (inst *QemuInstance) SSHAddress() (string, error) {
-	pid := fmt.Sprintf("%d", inst.Pid())
-	data, err := ioutil.ReadFile("/proc/net/tcp")
-	if err != nil {
-		return "", errors.Wrap(err, "reading /proc/net/tcp")
-	}
-
-	for _, line := range strings.Split(string(data), "\n")[1:] {
-		fields := strings.Fields(line)
-		if len(fields) < 10 {
-			// at least 10 fields are neeeded for the local & remote address and the inode
-			continue
-		}
-		localAddress := fields[1]
-		remoteAddress := fields[2]
-		inode := fields[9]
-
-		var isLocalPat *regexp.Regexp
-		if util.HostEndianness == util.LITTLE {
-			isLocalPat = regexp.MustCompile("0100007F:[[:xdigit:]]{4}")
-		} else {
-			isLocalPat = regexp.MustCompile("7F000001:[[:xdigit:]]{4}")
-		}
-
-		if !isLocalPat.MatchString(localAddress) || remoteAddress != "00000000:0000" {
-			continue
-		}
-
-		dir := fmt.Sprintf("/proc/%s/fd/", pid)
-		fds, err := ioutil.ReadDir(dir)
-		if err != nil {
-			return "", fmt.Errorf("listing %s: %v", dir, err)
-		}
-
-		for _, f := range fds {
-			link, err := os.Readlink(filepath.Join(dir, f.Name()))
-			if err != nil {
-				continue
-			}
-			socketPattern := regexp.MustCompile("socket:\\[([0-9]+)\\]")
-			match := socketPattern.FindStringSubmatch(link)
-			if len(match) > 1 {
-				if inode == match[1] {
-					// this entry belongs to the QEMU pid, parse the port and return the address
-					portHex := strings.Split(localAddress, ":")[1]
-					port, err := strconv.ParseInt(portHex, 16, 32)
-					if err != nil {
-						return "", errors.Wrapf(err, "decoding port %q", portHex)
-					}
-					return fmt.Sprintf("127.0.0.1:%d", port), nil
-				}
-			}
+	for _, fwdPorts := range inst.hostForwardedPorts {
+		if fwdPorts.Service == "ssh" {
+			return fmt.Sprintf("127.0.0.1:%d", fwdPorts.HostPort), nil
 		}
 	}
 	return "", fmt.Errorf("didn't find an address")
@@ -275,6 +233,9 @@ type QemuBuilder struct {
 	ignitionSet      bool
 	ignitionRendered bool
 
+	UsermodeNetworking        bool
+	requestedHostForwardPorts []HostForwardPort
+
 	finalized bool
 	diskId    uint
 	disks     []*Disk
@@ -375,17 +336,34 @@ func (builder *QemuBuilder) ConsoleToFile(path string) {
 	builder.Append("-display", "none", "-chardev", "file,id=log,path="+path, "-serial", "chardev:log")
 }
 
-func (builder *QemuBuilder) EnableUsermodeNetworking(forwardedPort uint) {
+func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) {
+	builder.UsermodeNetworking = true
+	builder.requestedHostForwardPorts = h
+}
+
+func (builder *QemuBuilder) usermodeNetworkingAssignPorts() error {
 	netdev := "user,id=eth0"
-	if forwardedPort != 0 {
-		netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:0-:%d", forwardedPort)
+	for i := range builder.requestedHostForwardPorts {
+		address := fmt.Sprintf(":%d", builder.requestedHostForwardPorts[i].HostPort)
+		// Possible race condition between getting the port here and using it
+		// with qemu -- trade off for simpler port management
+		l, err := net.Listen("tcp", address)
+		if err != nil {
+			return err
+		}
+		l.Close()
+		builder.requestedHostForwardPorts[i].HostPort = l.Addr().(*net.TCPAddr).Port
+		netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:%d-:%d",
+			builder.requestedHostForwardPorts[i].HostPort,
+			builder.requestedHostForwardPorts[i].GuestPort)
 	}
 
 	if builder.Hostname != "" {
 		netdev += fmt.Sprintf(",hostname=%s", builder.Hostname)
 	}
 
 	builder.Append("-netdev", netdev, "-device", virtio("net", "netdev=eth0"))
+	return nil
 }
 
 // Mount9p sets up a mount point from the host to guest.  To be replaced
@@ -986,6 +964,14 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 		}
 	}
 
+	// Handle Usermode Networking
+	if builder.UsermodeNetworking {
+		if err := builder.usermodeNetworkingAssignPorts(); err != nil {
+			return nil, err
+		}
+		inst.hostForwardedPorts = builder.requestedHostForwardPorts
+	}
+
 	// Handle Software TPM
 	if builder.Swtpm && builder.supportsSwtpm() {
 		swtpmSock := filepath.Join(builder.tempdir, "swtpm-sock")