From db5da317cb4b4fc39db2d22752dd9e2d19b71626 Mon Sep 17 00:00:00 2001
From: Emilien Macchi <emilien@redhat.com>
Date: Wed, 23 Feb 2022 15:32:22 -0500
Subject: [PATCH] virtual/openstack: configure VFIO with NIOMMU by default

As well known and well documented, in virtual deployments of Kubernetes such as running on top
of OpenStack where the underlying virtualization platform is KVM, it does not support a virtualized iommu,
the VFIO PCI driver needs to be loaded with a special flag.

This patch aims to do it by default in the virtual_plugin, since this
plugin is only used by OpenStack for now.

Note: it also changes `utils.LoadKernelModule` function to support
kernel module arguments. It also adds a note in `load-kmod.sh` about the
fact that a module won't be reloaded with potential new options if the
module is already loaded on the system.
---
 bindata/scripts/load-kmod.sh          |  7 ++++++-
 pkg/plugins/virtual/virtual_plugin.go | 10 ++++++++++
 pkg/utils/utils.go                    |  9 +++++----
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/bindata/scripts/load-kmod.sh b/bindata/scripts/load-kmod.sh
index 7f3d4dc58..17364d735 100755
--- a/bindata/scripts/load-kmod.sh
+++ b/bindata/scripts/load-kmod.sh
@@ -1,11 +1,16 @@
 #!/bin/sh
 # chroot /host/ modprobe $1
 kmod_name=$(tr "-" "_" <<< $1)
+kmod_args="${@:2}"
 chroot /host/ lsmod | grep "^$1" >& /dev/null
 
 if [ $? -eq 0 ]
 then
+        # NOTE: We do not check if the module is loaded with specific options
+        #       so a manual reload is required if the module is loaded with
+        #       new or different options.
+        echo "Module $kmod_name already loaded; no change will be applied..."
         exit 0 
 else
-        chroot /host/ modprobe $kmod_name
+        chroot /host/ modprobe $kmod_name $kmod_args
 fi
diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go
index e14a2c8e4..dbae02e63 100644
--- a/pkg/plugins/virtual/virtual_plugin.go
+++ b/pkg/plugins/virtual/virtual_plugin.go
@@ -84,6 +84,16 @@ func (p *VirtualPlugin) Apply() error {
 	glog.Infof("virtual-plugin Apply(): desiredState=%v", p.DesireState.Spec)
 
 	if p.LoadVfioDriver == loading {
+		// In virtual deployments of Kubernetes where the underlying virtualization platform does not support a virtualized iommu
+		// the VFIO PCI driver needs to be loaded with a special flag.
+		// This is the case for OpenStack deployments where the underlying virtualization platform is KVM.
+		// NOTE: if VFIO was already loaded for some reason, we will not try to load it again with the new options.
+		kernelArgs := "enable_unsafe_noiommu_mode=1"
+		if err := utils.LoadKernelModule("vfio", kernelArgs); err != nil {
+			glog.Errorf("virtual-plugin Apply(): fail to load vfio kmod: %v", err)
+			return err
+		}
+
 		if err := utils.LoadKernelModule("vfio_pci"); err != nil {
 			glog.Errorf("virtual-plugin Apply(): fail to load vfio_pci kmod: %v", err)
 			return err
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index af7a9c267..79d809e35 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -534,12 +534,13 @@ func getVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualF
 	return vf
 }
 
-func LoadKernelModule(name string) error {
-	glog.Infof("LoadKernelModule(): try to load kernel module %s", name)
-	cmd := exec.Command("/bin/sh", scriptsPath, name)
+func LoadKernelModule(name string, args ...string) error {
+	glog.Infof("LoadKernelModule(): try to load kernel module %s with arguments '%s'", name, args)
+	cmdArgs := strings.Join(args, " ")
+	cmd := exec.Command("/bin/sh", scriptsPath, name, cmdArgs)
 	err := cmd.Run()
 	if err != nil {
-		glog.Errorf("LoadKernelModule(): fail to load kernel module %s: %v", name, err)
+		glog.Errorf("LoadKernelModule(): fail to load kernel module %s with arguments '%s': %v", name, args, err)
 		return err
 	}
 	return nil