virtio-mmio: Add support for virtio-mmio

Start adding support for virtio-mmio devices starting with block.
The devices show within the vm as vda, vdb,... based on order of
insertion and such within the VM resemble virtio-blk devices.

They need to be explicitly differentiated to ensure that the
agent logic within the VM can discover and mount them appropropriately.
The agent uses PCI location to discover them for virtio-blk.
For virtio-mmio we need to use the predicted device name for now.

Fixes: kata-containers#1046

Signed-off-by: Manohar Castelino <[email protected]>

pkg/katautils/config.go

@@ -297,8 +297,8 @@ func (h hypervisor) blockDeviceDriver() (string, error) {
 		return defaultBlockDeviceDriver, nil
 	}
 
-	if h.BlockDeviceDriver != vc.VirtioSCSI && h.BlockDeviceDriver != vc.VirtioBlock {
-		return "", fmt.Errorf("Invalid value %s provided for hypervisor block storage driver, can be either %s or %s", h.BlockDeviceDriver, vc.VirtioSCSI, vc.VirtioBlock)
+	if h.BlockDeviceDriver != vc.VirtioSCSI && h.BlockDeviceDriver != vc.VirtioBlock && h.BlockDeviceDriver != vc.VirtioMmio {
+		return "", fmt.Errorf("Invalid value %s provided for hypervisor block storage driver, can be either %s or %s or %s", h.BlockDeviceDriver, vc.VirtioSCSI, vc.VirtioBlock, vc.VirtioMmio)
 	}
 
 	return h.BlockDeviceDriver, nil

virtcontainers/device/config/config.go

@@ -98,6 +98,9 @@ type BlockDrive struct {
 	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
 	Index int
 
+	// MmioAddr is used to identify the slot at which the drive is attached (order?).
+	MmioAddr string
+
 	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
 	PCIAddr string
 

virtcontainers/device/drivers/block.go

@@ -73,8 +73,13 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 	}
 
 	customOptions := device.DeviceInfo.DriverOptions
-	if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
-		drive.VirtPath = filepath.Join("/dev", driveName)
+	if customOptions != nil {
+		if customOptions["block-driver"] == "virtio-blk" {
+			drive.VirtPath = filepath.Join("/dev", driveName)
+		}
+		if customOptions["block-driver"] == "virtio-mmio" {
+			drive.VirtPath = filepath.Join("/dev", driveName)
+		}
 	} else {
 		scsiAddr, err := utils.GetSCSIAddress(index)
 		if err != nil {

virtcontainers/device/manager/manager.go

@@ -20,6 +20,8 @@ import (
 )
 
 const (
+	// VirtioMmio indicates block driver is virtio-mmio based
+	VirtioMmio string = "virtio-mmio"
 	// VirtioBlock indicates block driver is virtio-blk based
 	VirtioBlock string = "virtio-blk"
 	// VirtioSCSI indicates block driver is virtio-scsi based
@@ -55,7 +57,9 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
 	dm := &deviceManager{
 		devices: make(map[string]api.Device),
 	}
-	if blockDriver == VirtioBlock {
+	if blockDriver == VirtioMmio {
+		dm.blockDriver = VirtioMmio
+	} else if blockDriver == VirtioBlock {
 		dm.blockDriver = VirtioBlock
 	} else {
 		dm.blockDriver = VirtioSCSI

virtcontainers/hyperstart_agent.go

@@ -522,7 +522,8 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {
 
 	if c.state.Fstype != "" {
 		// Pass a drive name only in case of block driver
-		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+		if (sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock) ||
+			(sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioMmio) {
 			driveName, err := utils.GetVirtDriveName(c.state.BlockIndex)
 			if err != nil {
 				return err

virtcontainers/kata_agent.go

@@ -58,6 +58,7 @@ var (
 	// CAP_NET_BIND_SERVICE capability may bind to these port numbers.
 	vSockPort            = 1024
 	kata9pDevType        = "9p"
+	kataMmioBlkDevType   = "mmioblk"
 	kataBlkDevType       = "blk"
 	kataSCSIDevType      = "scsi"
 	sharedDir9pOptions   = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"}
@@ -845,7 +846,10 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 			ContainerPath: dev.ContainerPath,
 		}
 
-		if d.SCSIAddr == "" {
+		if d.VirtPath != "" {
+			kataDevice.Type = kataMmioBlkDevType
+			kataDevice.Id = d.VirtPath
+		} else if d.SCSIAddr == "" {
 			kataDevice.Type = kataBlkDevType
 			kataDevice.Id = d.PCIAddr
 		} else {
@@ -897,7 +901,10 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 			return nil, fmt.Errorf("malformed block drive")
 		}
 
-		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioMmio {
+			rootfs.Driver = kataMmioBlkDevType
+			rootfs.Source = blockDrive.MmioAddr
+		} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
 			rootfs.Driver = kataBlkDevType
 			rootfs.Source = blockDrive.PCIAddr
 		} else {
@@ -1103,6 +1110,9 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 		if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
 			vol.Driver = kataBlkDevType
 			vol.Source = blockDrive.PCIAddr
+		} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioMmio {
+			vol.Driver = kataMmioBlkDevType
+			vol.Source = blockDrive.VirtPath
 		} else {
 			vol.Driver = kataSCSIDevType
 			vol.Source = blockDrive.SCSIAddr

virtcontainers/qemu_arch_base.go

@@ -128,7 +128,11 @@ const (
 // 0 is reserved.
 const bridgePCIStartAddr = 2
 
+//TODO: This should be a hypervisor agnostic definition
 const (
+	// VirtioMmio means use virtio-mmio for mmio based drives
+	VirtioMmio = "virtio-mmio"
+
 	// VirtioBlock means use virtio-blk for hotplugging drives
 	VirtioBlock = "virtio-blk"