[Experiment] Explore running multiple containers in a shared VM

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -1901,7 +1901,8 @@
 		config.UUIDandVersion, config.DisplayName)
 	status.DiskStatusList = make([]types.DiskStatus,
 		len(config.DiskConfigList))
-	need9P := false
+	var addDiskStatus []types.DiskStatus
+	status.ContainerList = nil
 	for i, dc := range config.DiskConfigList {
 		ds := &status.DiskStatusList[i]
 		ds.VolumeKey = dc.VolumeKey
@@ -1917,12 +1918,31 @@
 		if dc.Target == zconfig.Target_AppCustom {
 			ds.Devtype = "AppCustom"
 		} else if dc.Format == zconfig.Format_CONTAINER {
-			if i == 0 {
+			if i == 0 || ds.MountDir == "" {
 				ds.MountDir = "/"
-				status.OCIConfigDir = ds.FileLocation
+				suffix := ""
+				if i != 0 {
+					suffix = strconv.Itoa(i)
+				}
+				fileLocation := "/mnt" + suffix
+				dcs := types.DomainContainerStatus{
+					ContainerIndex: i,
+					FileLocation:   fileLocation,
+					OCIConfigDir:   ds.FileLocation,
+				}
+				status.ContainerList = append(status.ContainerList, dcs)
+				// Add information about the 9P export to
+				// DiskStatus.
+				// XXX Why do we/did we reuse DIskStatus here?
+				// Note that Format can not be set to CONTAINER
+				addDiskStatus = append(addDiskStatus,
+					types.DiskStatus{
+						FileLocation: ds.FileLocation,
+						Devtype:      "9P",
+						ReadOnly:     false,
+					})
 			}
 			ds.Devtype = ""
-			need9P = true
 		} else {
 			ds.Devtype = "hdd"
 			if dc.Format == zconfig.Format_ISO {
@@ -1948,7 +1968,7 @@
 			return fmt.Errorf("failed to fetch cloud-init userdata: %s",
 				err)
 		}
-		if status.OCIConfigDir != "" { // If AppInstance is a container, we need to parse cloud-init config and apply the supported parts
+		if len(status.ContainerList) != 0 { // AppInstance is a container, we need to parse cloud-init config and apply the supported parts
 			if cloudconfig.IsCloudConfig(ciStr) { // treat like the cloud-init config
 				cc, err := cloudconfig.ParseCloudConfig(ciStr)
 				if err != nil {
@@ -1984,13 +2004,7 @@
 		}
 	}
 
-	if need9P {
-		status.DiskStatusList = append(status.DiskStatusList, types.DiskStatus{
-			FileLocation: "/mnt",
-			Devtype:      "9P",
-			ReadOnly:     false,
-		})
-	}
+	status.DiskStatusList = append(status.DiskStatusList, addDiskStatus...)
 	return nil
 }
 

pkg/pillar/containerd/oci.go

@@ -26,6 +26,8 @@
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/sirupsen/logrus"
 )
 
 const eveScript = "/bin/eve"
@@ -210,6 +212,7 @@
 	// finally do a switcheroo
 	s.Spec = spec.Spec
 
+	logrus.Infof("XXX AddLoader result %+v", s.Spec)
 	return nil
 }
 
@@ -440,6 +443,7 @@
 	blkMountPoints := ""
 
 	for id, disk := range disks {
+		// XXX here we use MountDir and FileLocation
 		dst := disk.MountDir
 		src := disk.FileLocation
 		opts := []string{"rbind"}
@@ -453,14 +457,20 @@
 		// we can bind mount anything aside from FmtUnknown
 		switch disk.Format {
 		case zconfig.Format_FmtUnknown:
+			logrus.Infof("XXX FmtUnknown %d", id)
 			continue
 		case zconfig.Format_CONTAINER:
+			logrus.Infof("XXX container %d MountDir %s",
+				id, disk.MountDir)
 			if path.Clean(disk.MountDir) == "/" {
 				// skipping root volumes for now
 				continue
 			} else {
 				src = path.Join(src, ociVolumeData)
 			}
+		default:
+			logrus.Infof("XXX format %s %d", disk.Format, id)
+
 		}
 
 		dests := []string{fmt.Sprintf("/dev/eve/volumes/by-id/%d", id)}
@@ -481,7 +491,10 @@
 			dests = append(dests, dst)
 		}
 
-		for _, dest := range dests {
+		for i, dest := range dests {
+			// XXX here we create bind mounts from FileLocation to MountDir
+			logrus.Infof("XXX Adding mount %d/%d from %s to %s",
+				id, i, src, dest)
 			s.Mounts = append(s.Mounts, specs.Mount{
 				Type:        "bind",
 				Source:      src,

pkg/pillar/hypervisor/containerd.go

@@ -95,15 +95,20 @@
 
 func (ctx ctrdContext) Setup(status types.DomainStatus, config types.DomainConfig,
 	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
-	if status.OCIConfigDir == "" {
+	if len(status.ContainerList) == 0 {
 		return logError("failed to run domain %s: not based on an OCI image", status.DomainName)
 	}
 
-	spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir)
+	// We assume a single container in the case of NOHYPER
+	if len(status.ContainerList) != 1 {
+		return logError("failed to run domain %s: more than one OCI image", status.DomainName)
+	}
+
+	ociConfigDir := status.ContainerList[0].OCIConfigDir
+	spec, err := ctx.setupSpec(&status, &config, ociConfigDir)
 	if err != nil {
 		return logError("setting up OCI spec for domain %s failed %v", status.DomainName, err)
 	}
-
 	// we use patched version of dhcpcd with /etc/resolv.conf.new
 	vifsTaskResolv := filepath.Join(vifsDir, status.DomainName, "etc", "resolv.conf.new")
 	err = os.MkdirAll(filepath.Dir(vifsTaskResolv), 0755)

pkg/pillar/hypervisor/kvm.go

@@ -8,6 +8,7 @@
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -261,7 +262,7 @@
 [device "fs{{.DiskID}}"]
   driver = "virtio-9p-pci"
   fsdev = "fsdev{{.DiskID}}"
-  mount_tag = "share_dir"
+  mount_tag = "{{.Tag9P}}"
   addr = "{{printf "0x%x" .PCIId}}"
 {{else}}
 [device "pci.{{.PCIId}}"]
@@ -673,24 +674,50 @@
 		"-readconfig", file.Name(),
 		"-pidfile", kvmStateDir+domainName+"/pid")
 
-	spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir)
-	if err != nil {
-		return logError("failed to load OCI spec for domain %s: %v", status.DomainName, err)
+	var spec0 containerd.OCISpec // The main container for the app instance
+	for _, dcs := range status.ContainerList {
+		ociConfigDir := dcs.OCIConfigDir
+		logrus.Infof("XXX processing %d dir %s", dcs.ContainerIndex, ociConfigDir)
+		spec, err := ctx.setupSpec(&status, &config, ociConfigDir)
+		if err != nil {
+			return logError("failed to load OCI spec for domain %s oci %d %s: %v",
+				status.DomainName, dcs.ContainerIndex, ociConfigDir, err)
+		}
+		if dcs.ContainerIndex == 0 {
+			spec0 = spec
+		}
+		if err = spec.AddLoader("/containers/services/xen-tools"); err != nil {
+			return logError("failed to add kvm hypervisor loader to domain %s oci %d %s: %v",
+				status.DomainName, dcs.ContainerIndex, ociConfigDir, err)
+		}
 	}
-	if err = spec.AddLoader("/containers/services/xen-tools"); err != nil {
-		return logError("failed to add kvm hypervisor loader to domain %s: %v", status.DomainName, err)
+	// XXX find better code structure
+	if spec0 == nil {
+		// Not a container
+		var err error
+		spec0, err = ctx.setupSpec(&status, &config, "")
+		if err != nil {
+			return logError("failed to load OCI spec for domain %s: %v",
+				status.DomainName, err)
+		}
+		if err = spec0.AddLoader("/containers/services/xen-tools"); err != nil {
+			return logError("failed to add kvm hypervisor loader to domain %s: %v",
+				status.DomainName, err)
+		}
 	}
+
+	// XXX potentially IoAdpaters per OCI? Would require different EVE API
 	overhead, err := vmmOverhead(domainName, domainUUID, int64(config.Memory), int64(config.VMMMaxMem), int64(config.MaxCpus), int64(config.VCpus), config.IoAdapterList, aa, globalConfig)
 	if err != nil {
 		return logError("vmmOverhead() failed for domain %s: %v",
 			status.DomainName, err)
 	}
 	logrus.Debugf("Qemu overhead for domain %s is %d bytes", status.DomainName, overhead)
-	spec.AdjustMemLimit(config, overhead)
-	spec.Get().Process.Args = args
+	spec0.AdjustMemLimit(config, overhead)
+	spec0.Get().Process.Args = args
 	logrus.Infof("Hypervisor args: %v", args)
 
-	if err := spec.CreateContainer(true); err != nil {
+	if err := spec0.CreateContainer(true); err != nil {
 		return logError("Failed to create container for task %s from %v: %v", status.DomainName, config, err)
 	}
 
@@ -700,13 +727,19 @@
 // CreateDomConfig creates a domain config (a qemu config file, typically named something like xen-%d.cfg)
 func (ctx KvmContext) CreateDomConfig(domainName string, config types.DomainConfig, status types.DomainStatus,
 	diskStatusList []types.DiskStatus, aa *types.AssignableAdapters, file *os.File) error {
+	var extra string
+	numOCI := len(status.ContainerList)
+	if numOCI > 1 {
+		extra = fmt.Sprintf(" max_oci=%d", numOCI-1)
+	}
 	tmplCtx := struct {
 		Machine string
 		types.DomainConfig
 		types.DomainStatus
 	}{ctx.devicemodel, config, status}
 	tmplCtx.DomainConfig.Memory = (config.Memory + 1023) / 1024
 	tmplCtx.DomainConfig.DisplayName = domainName
+	tmplCtx.DomainConfig.ExtraArgs += extra
 
 	// render global device model settings
 	t, _ := template.New("qemu").Parse(qemuConfTemplate)
@@ -719,6 +752,7 @@
 		Machine                          string
 		PCIId, DiskID, SATAId, NumQueues int
 		AioType                          string
+		Tag9P                            string
 		types.DiskStatus
 	}{Machine: ctx.devicemodel, PCIId: 4, DiskID: 0, SATAId: 0, AioType: "io_uring", NumQueues: config.VCpus}
 
@@ -727,14 +761,23 @@
 		Parse(qemuDiskTemplate)
 	for _, ds := range diskStatusList {
 		if ds.Devtype == "" {
+			// XXX doesn't all 9p fall in here?
+			logrus.Infof("XXX skipping DiskStatus: %+v", ds)
 			continue
 		}
 		if ds.Devtype == "AppCustom" {
 			// This is application custom data. It is forwarded to the VM
 			// differently - as a download url in zedrouter
 			continue
 		}
+		tag := "share_dir"
+		if diskContext.DiskID != 0 {
+			tag += strconv.Itoa(diskContext.DiskID)
+		}
+		diskContext.Tag9P = tag
 		diskContext.DiskStatus = ds
+		logrus.Infof("XXX tag %s DiskStatus %d: %+v",
+			tag, diskContext.DiskID, ds)
 		if err := t.Execute(file, diskContext); err != nil {
 			return logError("can't write to config file %s (%v)", file.Name(), err)
 		}