openstack: Cherry-pick from upstream

bindata/scripts/load-kmod.sh

@@ -1,11 +1,16 @@
 #!/bin/sh
 # chroot /host/ modprobe $1
 kmod_name=$(tr "-" "_" <<< $1)
+kmod_args="${@:2}"
 chroot /host/ lsmod | grep "^$1" >& /dev/null
 
 if [ $? -eq 0 ]
 then
+        # NOTE: We do not check if the module is loaded with specific options
+        #       so a manual reload is required if the module is loaded with
+        #       new or different options.
+        echo "Module $kmod_name already loaded; no change will be applied..."
         exit 0 
 else
-        chroot /host/ modprobe $kmod_name
+        chroot /host/ modprobe $kmod_name $kmod_args
 fi

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.5.2
+	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/intel/sriov-network-device-plugin v0.0.0-20200924101303-b7f6d3e06797
 	github.com/jaypipes/ghw v0.6.1
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc
@@ -82,6 +83,7 @@ require (
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/googleapis/gnostic v0.5.1 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect

go.sum

@@ -435,10 +435,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=

pkg/daemon/writer.go

@@ -43,6 +43,16 @@ func NewNodeStateStatusWriter(c snclientset.Interface, n string, f func()) *Node
 func (writer *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message, syncCh chan<- struct{}, destDir string, runonce bool, platformType utils.PlatformType) {
 	glog.V(0).Infof("Run(): start writer")
 	msg := Message{}
+
+	var err error
+
+	if platformType == utils.VirtualOpenStack {
+		writer.metaData, writer.networkData, err = utils.GetOpenstackData()
+		if err != nil {
+			glog.Errorf("Run(): failed to get OpenStack data: %v", err)
+		}
+	}
+
 	if runonce {
 		glog.V(0).Info("Run(): once")
 		if err := writer.pollNicStatus(platformType); err != nil {

pkg/plugins/virtual/virtual_plugin.go

@@ -84,6 +84,16 @@ func (p *VirtualPlugin) Apply() error {
 	glog.Infof("virtual-plugin Apply(): desiredState=%v", p.DesireState.Spec)
 
 	if p.LoadVfioDriver == loading {
+		// In virtual deployments of Kubernetes where the underlying virtualization platform does not support a virtualized iommu
+		// the VFIO PCI driver needs to be loaded with a special flag.
+		// This is the case for OpenStack deployments where the underlying virtualization platform is KVM.
+		// NOTE: if VFIO was already loaded for some reason, we will not try to load it again with the new options.
+		kernelArgs := "enable_unsafe_noiommu_mode=1"
+		if err := utils.LoadKernelModule("vfio", kernelArgs); err != nil {
+			glog.Errorf("virtual-plugin Apply(): fail to load vfio kmod: %v", err)
+			return err
+		}
+
 		if err := utils.LoadKernelModule("vfio_pci"); err != nil {
 			glog.Errorf("virtual-plugin Apply(): fail to load vfio_pci kmod: %v", err)
 			return err

pkg/utils/utils.go

@@ -534,12 +534,13 @@ func getVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualF
 	return vf
 }
 
-func LoadKernelModule(name string) error {
-	glog.Infof("LoadKernelModule(): try to load kernel module %s", name)
-	cmd := exec.Command("/bin/sh", scriptsPath, name)
+func LoadKernelModule(name string, args ...string) error {
+	glog.Infof("LoadKernelModule(): try to load kernel module %s with arguments '%s'", name, args)
+	cmdArgs := strings.Join(args, " ")
+	cmd := exec.Command("/bin/sh", scriptsPath, name, cmdArgs)
 	err := cmd.Run()
 	if err != nil {
-		glog.Errorf("LoadKernelModule(): fail to load kernel module %s: %v", name, err)
+		glog.Errorf("LoadKernelModule(): fail to load kernel module %s with arguments '%s': %v", name, args, err)
 		return err
 	}
 	return nil

pkg/utils/utils_virtual.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"os"
 	"strconv"
 
 	"github.com/golang/glog"
+	"github.com/hashicorp/go-retryablehttp"
 	dputils "github.com/intel/sriov-network-device-plugin/pkg/utils"
 	"github.com/jaypipes/ghw"
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
@@ -42,9 +44,12 @@ var (
 )
 
 const (
-	ospMetaDataDir = "/host/var/config/openstack/latest/"
-	ospNetworkData = ospMetaDataDir + "/network_data.json"
-	ospMetaData    = ospMetaDataDir + "/meta_data.json"
+	ospMetaDataDir     = "/host/var/config/openstack/2018-08-27"
+	ospMetaDataBaseUrl = "http://169.254.169.254/openstack/2018-08-27"
+	ospNetworkDataFile = ospMetaDataDir + "/network_data.json"
+	ospMetaDataFile    = ospMetaDataDir + "/meta_data.json"
+	ospNetworkDataUrl  = ospMetaDataBaseUrl + "/network_data.json"
+	ospMetaDataUrl     = ospMetaDataBaseUrl + "/meta_data.json"
 )
 
 // OSPMetaDataDevice -- Device structure within meta_data.json
@@ -93,47 +98,89 @@ type OSPNetworkData struct {
 	// Omit Services
 }
 
-func metaData(platformType PlatformType, address string) (netFilter string, macAddress string) {
-	switch platformType {
-	case VirtualOpenStack:
-		metaData, networkData := readOpenstackMetaData()
-		netFilter, macAddress = parseOpenstackMetaData(address, metaData, networkData)
-
-	default:
-		glog.V(2).Infof("Unknown PlatformType: %v", platformType)
+// GetOpenstackData gets the metadata and network_data
+func GetOpenstackData() (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
+	metaData, networkData, err = getOpenstackDataFromConfigDrive()
+	if err != nil {
+		metaData, networkData, err = getOpenstackDataFromMetadataService()
 	}
-
-	return
+	return metaData, networkData, err
 }
 
-func readOpenstackMetaData() (metaData *OSPMetaData, networkData *OSPNetworkData) {
+// getOpenstackDataFromConfigDrive reads the meta_data and network_data files
+func getOpenstackDataFromConfigDrive() (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
+	metaData = &OSPMetaData{}
 	networkData = &OSPNetworkData{}
+	glog.Infof("reading OpenStack meta_data from config-drive")
+	var metadataf *os.File
+	metadataf, err = os.Open(ospMetaDataFile)
+	if err != nil {
+		return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospMetaDataFile, err)
+	}
+	defer func() {
+		if e := metadataf.Close(); err == nil && e != nil {
+			err = fmt.Errorf("error closing file %s: %w", ospMetaDataFile, e)
+		}
+	}()
+	if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil {
+		return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospMetaDataFile, err)
+	}
 
-	rawBytes, err := ioutil.ReadFile(ospNetworkData)
+	glog.Infof("reading OpenStack network_data from config-drive")
+	var networkDataf *os.File
+	networkDataf, err = os.Open(ospNetworkDataFile)
 	if err != nil {
-		glog.Errorf("error reading file %s, %v", ospNetworkData, err)
-		return
+		return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospNetworkDataFile, err)
 	}
+	defer func() {
+		if e := networkDataf.Close(); err == nil && e != nil {
+			err = fmt.Errorf("error closing file %s: %w", ospNetworkDataFile, e)
+		}
+	}()
+	if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil {
+		return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospNetworkDataFile, err)
+	}
+	return metaData, networkData, err
+}
 
-	if err = json.Unmarshal(rawBytes, networkData); err != nil {
-		glog.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkData)
-		return
+func getBodyFromUrl(url string) ([]byte, error) {
+	glog.V(2).Infof("Getting body from %s", url)
+	resp, err := retryablehttp.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	rawBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
 	}
+	defer resp.Body.Close()
+	return rawBytes, nil
+}
 
+// getOpenstackDataFromMetadataService fetchs the metadata and network_data from the metadata service
+func getOpenstackDataFromMetadataService() (metaData *OSPMetaData, networkData *OSPNetworkData, err error) {
 	metaData = &OSPMetaData{}
-
-	rawBytes, err = ioutil.ReadFile(ospMetaData)
+	networkData = &OSPNetworkData{}
+	glog.Infof("getting OpenStack meta_data from metadata server")
+	metaDataRawBytes, err := getBodyFromUrl(ospMetaDataUrl)
 	if err != nil {
-		glog.Errorf("error reading file %s, %v", ospMetaData, err)
-		return
+		return metaData, networkData, fmt.Errorf("error getting OpenStack meta_data from %s: %v", ospMetaDataUrl, err)
 	}
-
-	if err = json.Unmarshal(rawBytes, metaData); err != nil {
-		glog.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkData)
-		return
+	err = json.Unmarshal(metaDataRawBytes, metaData)
+	if err != nil {
+		return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospMetaDataUrl)
 	}
 
-	return
+	glog.Infof("getting OpenStack network_data from metadata server")
+	networkDataRawBytes, err := getBodyFromUrl(ospNetworkDataUrl)
+	if err != nil {
+		return metaData, networkData, fmt.Errorf("error getting OpenStack network_data from %s: %v", ospNetworkDataUrl, err)
+	}
+	err = json.Unmarshal(networkDataRawBytes, networkData)
+	if err != nil {
+		return metaData, networkData, fmt.Errorf("error unmarshalling raw bytes %v from %s", err, ospNetworkDataUrl)
+	}
+	return metaData, networkData, nil
 }
 
 func parseOpenstackMetaData(pciAddr string, metaData *OSPMetaData, networkData *OSPNetworkData) (networkID string, macAddress string) {