Support changing Eswitch mode of SR-IOV NICs for kubernetes deployment

Signed-off-by: Mamduh Alassi <[email protected]>

Makefile

@@ -63,7 +63,7 @@ _build-%:
 _plugin-%: vet
 	@hack/build-plugins.sh $*
 
-plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco
+plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s
 
 clean:
 	@rm -rf $(TARGET_DIR)

bindata/manifests/daemon/daemonset.yaml

@@ -50,6 +50,10 @@ spec:
         volumeMounts:
         - name: host
           mountPath: /host
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh","-c","rm -f /host/etc/systemd/system/switchdev-configuration.service"]
       volumes:
       - name: host
         hostPath:

go.mod

@@ -29,6 +29,7 @@ require (
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.19.0
 	k8s.io/apimachinery v0.19.0
 	k8s.io/client-go v0.19.0

pkg/daemon/daemon.go

@@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er
 		pl = registerPlugins(ns)
 		if utils.ClusterType == utils.ClusterTypeOpenshift {
 			pl = append(pl, McoPlugin)
+		} else {
+			pl = append(pl, K8sPlugin)
 		}
 		pl = append(pl, GenericPlugin)
 	}

pkg/daemon/plugin.go

@@ -31,6 +31,7 @@ const (
 	GenericPlugin = "generic_plugin"
 	VirtualPlugin = "virtual_plugin"
 	McoPlugin     = "mco_plugin"
+	K8sPlugin     = "k8s_plugin"
 )
 
 // loadPlugin loads a single plugin from a file path

pkg/plugins/k8s/k8s_plugin.go

@@ -0,0 +1,290 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/coreos/go-systemd/v22/unit"
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
+)
+
+type K8sPlugin struct {
+	PluginName  string
+	SpecVersion string
+}
+
+type McoService struct {
+	name     string
+	Contents string
+}
+
+type McoFile struct {
+	Path     string
+	Contents struct {
+		Inline string
+	}
+}
+
+type McoServiceInjection struct {
+	Name    string
+	Dropins []struct {
+		Contents string
+	}
+}
+
+type k8sUpdateTarget struct {
+	switchdevService bool
+	switchdevScript  bool
+	networkManager   bool
+}
+
+func (u *k8sUpdateTarget) needUpdate() bool {
+	return u.switchdevService || u.switchdevScript || u.networkManager
+}
+
+const (
+	mcoManifestPath           = "bindata/manifests/machine-config/"
+	switchdevUnits            = mcoManifestPath + "switchdev-units/"
+	switchdevUnitFile         = switchdevUnits + "switchdev-configuration.yaml"
+	networkManagerUnitFile    = switchdevUnits + "NetworkManager.service.yaml"
+	configuresSwitchdevScript = mcoManifestPath + "files/configure-switchdev.sh.yaml"
+
+	systemdDir = "/usr/lib/systemd/system/"
+	chroot     = "/host"
+)
+
+var (
+	Plugin K8sPlugin
+
+	serviceManager service.ServiceManager
+	updateTarget   *k8sUpdateTarget
+
+	switchdevRunScript    *McoFile
+	switchdevService      *service.Service
+	networkManagerService *service.Service
+)
+
+// Initialize our plugin and set up initial values
+func init() {
+	Plugin = K8sPlugin{
+		PluginName:  "k8s_plugin",
+		SpecVersion: "1.0",
+	}
+
+	serviceManager = service.NewServiceManager(chroot)
+}
+
+// Name returns the name of the plugin
+func (p *K8sPlugin) Name() string {
+	return p.PluginName
+}
+
+// Spec returns the version of the spec expected by the plugin
+func (p *K8sPlugin) Spec() string {
+	return p.SpecVersion
+}
+
+// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateAdd()")
+	return p.OnNodeStateChange(nil, state)
+}
+
+// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateChange()")
+	needDrain = false
+	needReboot = false
+
+	err = p.readMcoFiles()
+	if err != nil {
+		return
+	}
+
+	// Check services
+	updateTarget, err = handleServices()
+	if err != nil {
+		return
+	}
+
+	// Check switchdev config
+	var update, remove bool
+	if new == nil {
+		return
+	}
+	if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil {
+		glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err)
+		return
+	}
+	if remove {
+		glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs")
+		needDrain = true
+		needReboot = true
+		return
+	}
+	if update {
+		glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf")
+		needDrain = true
+		needReboot = true
+		return
+	}
+	if updateTarget.needUpdate() {
+		glog.Infof("k8s-plugin OnNodeStateChange(): needDrain to update %+v", updateTarget)
+		needDrain = true
+	}
+
+	return
+}
+
+// Apply config change
+func (p *K8sPlugin) Apply() error {
+	glog.Info("k8s-plugin Apply()")
+	if updateTarget.switchdevService {
+		err := serviceManager.EnableService(switchdevService)
+		if err != nil {
+			return err
+		}
+	}
+
+	if updateTarget.switchdevScript {
+		err := ioutil.WriteFile(path.Join(chroot, switchdevRunScript.Path),
+			[]byte(switchdevRunScript.Contents.Inline), 0755)
+		if err != nil {
+			return err
+		}
+	}
+
+	if updateTarget.networkManager {
+		nmService, err := serviceManager.ReadService(networkManagerService.Path)
+		if err != nil {
+			return err
+		}
+		if nmService == nil {
+			// Invalid case to reach here
+			return fmt.Errorf("k8s-plugin Apply(): can't update non-existing service \"NetworkManager\"")
+		}
+		service, err := service.ServiceAppend(nmService, networkManagerService)
+		if err != nil {
+			return err
+		}
+
+		err = serviceManager.EnableService(service)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *K8sPlugin) readMcoFiles() error {
+	if switchdevService == nil {
+		data, err := ioutil.ReadFile(switchdevUnitFile)
+		if err != nil {
+			return err
+		}
+
+		var switchdevServiceMco *McoService
+		if err := yaml.Unmarshal(data, &switchdevServiceMco); err != nil {
+			return err
+		}
+
+		switchdevService := &service.Service{
+			Name:    switchdevServiceMco.name,
+			Path:    "/etc/systemd/system/" + switchdevServiceMco.name,
+			Content: switchdevServiceMco.Contents,
+		}
+		// Remove run condition form the service
+		conditionOpt := &unit.UnitOption{
+			Section: "unit",
+			Name:    "ConditionPathExists",
+			Value:   "!/etc/ignition-machine-config-encapsulated.json",
+		}
+		switchdevService, err = service.RemoveFromService(switchdevService, conditionOpt)
+		if err != nil {
+			return err
+		}
+	}
+
+	if switchdevRunScript == nil {
+		data, err := ioutil.ReadFile(configuresSwitchdevScript)
+		if err != nil {
+			return err
+		}
+		if err := yaml.Unmarshal(data, &switchdevRunScript); err != nil {
+			return err
+		}
+		// Todo append running switchdev
+	}
+
+	if networkManagerService == nil {
+		data, err := ioutil.ReadFile(networkManagerUnitFile)
+		if err != nil {
+			return err
+		}
+		var networkManagerServiceContent McoServiceInjection
+		if err := yaml.Unmarshal(data, &networkManagerServiceContent); err != nil {
+			return err
+		}
+		networkManagerService = &service.Service{
+			Name:    networkManagerServiceContent.Name,
+			Path:    systemdDir + networkManagerServiceContent.Name,
+			Content: networkManagerServiceContent.Dropins[0].Contents,
+		}
+	}
+
+	return nil
+}
+
+func handleServices() (*k8sUpdateTarget, error) {
+	updateTarget := &k8sUpdateTarget{}
+
+	// Check switchdev service
+	swdService, err := serviceManager.ReadService(switchdevService.Path)
+	if err != nil {
+		return nil, err
+	}
+	if swdService == nil {
+		// service not exists
+		updateTarget.switchdevService = true
+	} else {
+		needChange, err := service.CompareServices(swdService, switchdevService)
+		if err != nil {
+			return nil, err
+		}
+		updateTarget.switchdevService = needChange
+	}
+
+	// Check switchdev run script
+	data, err := ioutil.ReadFile(path.Join(chroot, switchdevRunScript.Path))
+	if err != nil {
+		if os.IsNotExist(err) {
+			updateTarget.switchdevScript = true
+		}
+		return nil, err
+	} else if string(data) != switchdevRunScript.Contents.Inline {
+		updateTarget.switchdevScript = true
+	}
+
+	// Check Network Manager
+	nmService, err := serviceManager.ReadService(networkManagerService.Path)
+	if err != nil {
+		return nil, err
+	}
+	if nmService != nil {
+		needChange, err := service.CompareServices(nmService, networkManagerService)
+		if err != nil {
+			return nil, err
+		}
+		updateTarget.networkManager = needChange
+	}
+
+	return updateTarget, nil
+}