Support changing Eswitch mode of SR-IOV NICs for kubernetes deployment

Signed-off-by: Mamduh Alassi <[email protected]>

Makefile

@@ -63,7 +63,7 @@ _build-%:
 _plugin-%: vet
 	@hack/build-plugins.sh $*
 
-plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco
+plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s
 
 clean:
 	@rm -rf $(TARGET_DIR)

bindata/manifests/daemon/daemonset.yaml

@@ -50,6 +50,13 @@ spec:
         volumeMounts:
         - name: host
           mountPath: /host
+        lifecycle:
+          postStart:
+            exec:
+              command: ["/bin/sh", "-c", "touch /host/etc/sriov-netowrk-operator-run"]
+          preStop:
+            exec:
+              command: ["/bin/sh","-c","rm -f /host//etc/sriov-netowrk-operator-run"]
       volumes:
       - name: host
         hostPath:

bindata/manifests/k8s-services/ovs-vswitchd.service

@@ -0,0 +1,2 @@
+[Service]
+ExecStart=/bin/ovs-vsctl set Open_vSwitch . other_config:hw-offload=true

bindata/manifests/k8s-services/switchdev-configuration.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Configures SRIOV NIC into switchdev mode
+# Run service only when SR-IOV Network Operator is running
+ConditionPathExists=/etc/sriov-netowrk-operator-run
+# This service is used to move a SRIOV NIC into switchdev mode
+Wants=network-pre.target
+Before=network-pre.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/configure-switchdev.sh
+StandardOutput=journal+console
+StandardError=journal+console
+
+[Install]
+WantedBy=network-online.target

go.mod

@@ -29,6 +29,7 @@ require (
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.19.0
 	k8s.io/apimachinery v0.19.0
 	k8s.io/client-go v0.19.0

pkg/daemon/daemon.go

@@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er
 		pl = registerPlugins(ns)
 		if utils.ClusterType == utils.ClusterTypeOpenshift {
 			pl = append(pl, McoPlugin)
+		} else {
+			pl = append(pl, K8sPlugin)
 		}
 		pl = append(pl, GenericPlugin)
 	}

pkg/daemon/plugin.go

@@ -31,6 +31,7 @@ const (
 	GenericPlugin = "generic_plugin"
 	VirtualPlugin = "virtual_plugin"
 	McoPlugin     = "mco_plugin"
+	K8sPlugin     = "k8s_plugin"
 )
 
 // loadPlugin loads a single plugin from a file path

pkg/plugins/k8s/k8s_plugin.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"io/ioutil"
+	"path"
+
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
+)
+
+type K8sPlugin struct {
+	PluginName  string
+	SpecVersion string
+	services    []serviceConf
+}
+
+type serviceConf struct {
+	name       string
+	serviceBin string
+}
+
+const (
+	k8sServicesManifestPath   = "bindata/manifests/k8s-services"
+	ovsVSwitchdService        = "ovs-vswitchd"
+	switchdevService          = "switchdev-configuration"
+	configuresSwitchdevScript = "bindata/manifests/machine-config/files/configure-switchdev.sh.yaml"
+)
+
+var (
+	Plugin K8sPlugin
+
+	serviceManager service.ServiceManager
+)
+
+// Initialize our plugin and set up initial values
+func init() {
+	Plugin = K8sPlugin{
+		PluginName:  "k8s_plugin",
+		SpecVersion: "1.0",
+		services: []serviceConf{{
+			name:       "switchdev-configuration",
+			serviceBin: "configure-switchdev.sh",
+		}},
+	}
+	serviceManager = service.NewServiceManager("/host")
+}
+
+// Name returns the name of the plugin
+func (p *K8sPlugin) Name() string {
+	return p.PluginName
+}
+
+// Spec returns the version of the spec expected by the plugin
+func (p *K8sPlugin) Spec() string {
+	return p.SpecVersion
+}
+
+// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateAdd()")
+	return p.OnNodeStateChange(nil, state)
+}
+
+// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateChange()")
+	needDrain = false
+	needReboot = false
+
+	// Check services
+	var needServices bool
+	needServices, err = handleAddingServices()
+	if err != nil {
+		return
+	}
+
+	if !needServices {
+		needServices, err = serviceManager.CheckServicesLoaded()
+		if err != nil {
+			return
+		}
+	}
+	needDrain = needServices
+
+	// Check switchdev config
+	var update, remove bool
+	if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil {
+		glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err)
+		return
+	}
+	if remove {
+		needDrain = true
+		return
+	}
+	if update {
+		glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf")
+		needDrain = true
+		needReboot = true
+		return
+	}
+	return
+}
+
+// Apply config change
+func (p *K8sPlugin) Apply() error {
+	glog.Info("k8s-plugin Apply()")
+	if err := serviceManager.LoadMissingServices(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func handleAddingServices() (bool, error) {
+	needChange := false
+
+	smServices := serviceManager.ServicesSet()
+	if _, ok := smServices[ovsVSwitchdService]; !ok {
+		ovsService, err := service.ServiceFromFile(ovsVSwitchdService, path.Join(k8sServicesManifestPath, ovsVSwitchdService+".service"))
+		if err != nil {
+			return false, err
+		}
+		needChange = true
+		_ = serviceManager.AddService(ovsService)
+	}
+
+	if _, ok := smServices[switchdevService]; !ok {
+		switchdevService, err := service.ServiceFromFile(switchdevService, path.Join(k8sServicesManifestPath, switchdevService+".service"))
+		if err != nil {
+			return false, err
+		}
+		data, err := ioutil.ReadFile(configuresSwitchdevScript)
+		if err != nil {
+			return false, err
+		}
+
+		var switchdevBinYaml struct {
+			Path     string
+			Contents struct {
+				Inline string
+			}
+		}
+		if err := yaml.Unmarshal(data, &switchdevBinYaml); err != nil {
+			return false, err
+		}
+		switchdevService.Run = switchdevBinYaml.Path
+		switchdevService.RunBody = switchdevBinYaml.Contents.Inline
+		needChange = true
+		_ = serviceManager.AddService(switchdevService)
+	}
+
+	return needChange, nil
+}

pkg/plugins/mco/mco_plugin.go

@@ -3,8 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"os"
 
 	"github.com/golang/glog"
@@ -17,6 +15,7 @@ import (
 
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/controllers"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
 )
 
 type McoPlugin struct {
@@ -28,7 +27,6 @@ type McoPlugin struct {
 
 const (
 	switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service"
-	switchDevConfPath = "/host/etc/switchdev.conf"
 	nodeLabelPrefix   = "node-role.kubernetes.io/"
 )
 
@@ -89,7 +87,7 @@ func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS
 	}
 
 	var update, remove bool
-	if update, remove, err = writeSwitchdevConfFile(new); err != nil {
+	if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil {
 		glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err)
 		return
 	}
@@ -156,46 +154,3 @@ func (p *McoPlugin) Apply() error {
 	glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name)
 	return nil
 }
-
-func writeSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) {
-	_, err = os.Stat(switchDevConfPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			glog.V(2).Infof("writeSwitchdevConfFile(): file not existed, create it")
-			_, err = os.Create(switchDevConfPath)
-			if err != nil {
-				glog.Errorf("writeSwitchdevConfFile(): fail to create file: %v", err)
-				return
-			}
-		} else {
-			return
-		}
-	}
-	newContent := ""
-	for _, iface := range newState.Spec.Interfaces {
-		if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV {
-			newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs)
-		}
-	}
-	oldContent, err := ioutil.ReadFile(switchDevConfPath)
-	if err != nil {
-		glog.Errorf("writeSwitchdevConfFile(): fail to read file: %v", err)
-		return
-	}
-	if newContent == string(oldContent) {
-		glog.V(2).Info("writeSwitchdevConfFile(): no update")
-		return
-	}
-	if newContent == "" {
-		remove = true
-		glog.V(2).Info("writeSwitchdevConfFile(): remove content in switchdev.conf")
-	}
-	update = true
-	glog.V(2).Infof("writeSwitchdevConfFile(): write %s to switchdev.conf", newContent)
-	err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666)
-	if err != nil {
-		glog.Errorf("writeSwitchdevConfFile(): fail to write file: %v", err)
-		return
-	}
-	return
-}

pkg/service/Service.go

@@ -0,0 +1,26 @@
+package service
+
+import (
+	"io/ioutil"
+)
+
+type Service struct {
+	Name    string
+	Body    string
+	Run     string
+	RunBody string
+}
+
+func ServiceFromFile(name, file string) (*Service, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	service := &Service{
+		Name: name,
+		Body: string(data),
+	}
+
+	return service, nil
+}