Support changing Eswitch mode of SR-IOV NICs for kubernetes deployment

Signed-off-by: Mamduh Alassi <[email protected]>

Makefile

@@ -63,7 +63,7 @@ _build-%:
 _plugin-%: vet
 	@hack/build-plugins.sh $*
 
-plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco
+plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s
 
 clean:
 	@rm -rf $(TARGET_DIR)

bindata/manifests/daemon/daemonset.yaml

@@ -50,6 +50,10 @@ spec:
         volumeMounts:
         - name: host
           mountPath: /host
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh","-c","rm -f /host/etc/systemd/system/switchdev-configuration.service"]
       volumes:
       - name: host
         hostPath:

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/coreos/go-systemd/v22 v22.0.0
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-logr/logr v0.2.1
 	github.com/go-logr/zapr v0.2.0 // indirect
@@ -29,6 +30,7 @@ require (
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.19.0
 	k8s.io/apimachinery v0.19.0
 	k8s.io/client-go v0.19.0

pkg/daemon/daemon.go

@@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er
 		pl = registerPlugins(ns)
 		if utils.ClusterType == utils.ClusterTypeOpenshift {
 			pl = append(pl, McoPlugin)
+		} else {
+			pl = append(pl, K8sPlugin)
 		}
 		pl = append(pl, GenericPlugin)
 	}

pkg/daemon/plugin.go

@@ -31,6 +31,7 @@ const (
 	GenericPlugin = "generic_plugin"
 	VirtualPlugin = "virtual_plugin"
 	McoPlugin     = "mco_plugin"
+	K8sPlugin     = "k8s_plugin"
 )
 
 // loadPlugin loads a single plugin from a file path

pkg/plugins/k8s/k8s_plugin.go

@@ -0,0 +1,336 @@
+package main
+
+import (
+	"fmt"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/coreos/go-systemd/v22/unit"
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service"
+)
+
+type K8sPlugin struct {
+	PluginName  string
+	SpecVersion string
+}
+
+type McoService struct {
+	Name     string
+	Contents string
+}
+
+type McoFile struct {
+	Path     string
+	Contents struct {
+		Inline string
+	}
+}
+
+type McoServiceInjection struct {
+	Name    string
+	Dropins []struct {
+		Contents string
+	}
+}
+
+type k8sUpdateTarget struct {
+	switchdevService bool
+	switchdevScript  bool
+	networkManager   bool
+	openVSwitch      bool
+}
+
+func (u *k8sUpdateTarget) needUpdate() bool {
+	return u.switchdevService || u.switchdevScript || u.networkManager || u.openVSwitch
+}
+
+const (
+	mcoManifestPath           = "bindata/manifests/machine-config/"
+	switchdevUnits            = mcoManifestPath + "switchdev-units/"
+	switchdevUnitFile         = switchdevUnits + "switchdev-configuration.yaml"
+	networkManagerUnitFile    = switchdevUnits + "NetworkManager.service.yaml"
+	ovsUnitFile               = mcoManifestPath + "ovs-units/ovs-vswitchd.service.yaml"
+	configuresSwitchdevScript = mcoManifestPath + "files/configure-switchdev.sh.yaml"
+
+	systemdDir = "/usr/lib/systemd/system/"
+	chroot     = "/host"
+)
+
+var (
+	Plugin K8sPlugin
+
+	serviceManager service.ServiceManager
+	updateTarget   *k8sUpdateTarget
+
+	switchdevRunScript    *McoFile
+	switchdevService      *service.Service
+	openVSwitchService    *service.Service
+	networkManagerService *service.Service
+)
+
+// Initialize our plugin and set up initial values
+func init() {
+	Plugin = K8sPlugin{
+		PluginName:  "k8s_plugin",
+		SpecVersion: "1.0",
+	}
+
+	serviceManager = service.NewServiceManager(chroot)
+}
+
+// Name returns the name of the plugin
+func (p *K8sPlugin) Name() string {
+	return p.PluginName
+}
+
+// Spec returns the version of the spec expected by the plugin
+func (p *K8sPlugin) Spec() string {
+	return p.SpecVersion
+}
+
+// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateAdd()")
+	return p.OnNodeStateChange(nil, state)
+}
+
+// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node
+func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
+	glog.Info("k8s-plugin OnNodeStateChange()")
+	needDrain = false
+	needReboot = false
+
+	err = p.readMcoFiles()
+	if err != nil {
+		return
+	}
+
+	// Check services
+	updateTarget, err = handleServices()
+	if err != nil {
+		return
+	}
+
+	// Check switchdev config
+	var update, remove bool
+	if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil {
+		glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err)
+		return
+	}
+	if remove {
+		glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs")
+		needDrain = true
+		needReboot = true
+		return
+	}
+	if update {
+		glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf")
+		needDrain = true
+		needReboot = true
+		return
+	}
+	if updateTarget.needUpdate() {
+		glog.Infof("k8s-plugin OnNodeStateChange(): needDrain to update %+v", updateTarget)
+		needDrain = true
+	}
+
+	return
+}
+
+// Apply config change
+func (p *K8sPlugin) Apply() error {
+	glog.Info("k8s-plugin Apply()")
+	if updateTarget.switchdevService {
+		err := serviceManager.EnableService(switchdevService)
+		if err != nil {
+			return err
+		}
+	}
+
+	if updateTarget.switchdevScript {
+		err := ioutil.WriteFile(path.Join(chroot, switchdevRunScript.Path),
+			[]byte(switchdevRunScript.Contents.Inline), 0755)
+		if err != nil {
+			return err
+		}
+	}
+
+	if updateTarget.networkManager {
+		if err := updateService(networkManagerService); err != nil {
+			return err
+		}
+	}
+
+	if updateTarget.openVSwitch {
+		if err := updateService(openVSwitchService); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *K8sPlugin) readMcoFiles() error {
+	if switchdevService == nil {
+		data, err := ioutil.ReadFile(switchdevUnitFile)
+		if err != nil {
+			return err
+		}
+
+		var switchdevServiceMco *McoService
+		if err := yaml.Unmarshal(data, &switchdevServiceMco); err != nil {
+			return err
+		}
+
+		switchdevService = &service.Service{
+			Name:    switchdevServiceMco.Name,
+			Path:    "/etc/systemd/system/" + switchdevServiceMco.Name,
+			Content: switchdevServiceMco.Contents,
+		}
+		// Remove run condition form the service
+		conditionOpt := &unit.UnitOption{
+			Section: "Unit",
+			Name:    "ConditionPathExists",
+			Value:   "!/etc/ignition-machine-config-encapsulated.json",
+		}
+		switchdevService, err = service.RemoveFromService(switchdevService, conditionOpt)
+		if err != nil {
+			return err
+		}
+	}
+
+	if switchdevRunScript == nil {
+		data, err := ioutil.ReadFile(configuresSwitchdevScript)
+		if err != nil {
+			return err
+		}
+		if err := yaml.Unmarshal(data, &switchdevRunScript); err != nil {
+			return err
+		}
+	}
+
+	if networkManagerService == nil {
+		data, err := ioutil.ReadFile(networkManagerUnitFile)
+		if err != nil {
+			return err
+		}
+		var networkManagerServiceContent McoServiceInjection
+		if err := yaml.Unmarshal(data, &networkManagerServiceContent); err != nil {
+			return err
+		}
+		networkManagerService = &service.Service{
+			Name:    networkManagerServiceContent.Name,
+			Path:    systemdDir + networkManagerServiceContent.Name,
+			Content: networkManagerServiceContent.Dropins[0].Contents,
+		}
+	}
+
+	if openVSwitchService == nil {
+		data, err := ioutil.ReadFile(ovsUnitFile)
+		if err != nil {
+			return err
+		}
+		var ovsServiceContent McoServiceInjection
+		if err := yaml.Unmarshal(data, &ovsServiceContent); err != nil {
+			return err
+		}
+		openVSwitchService = &service.Service{
+			Name:    ovsServiceContent.Name,
+			Path:    systemdDir + ovsServiceContent.Name,
+			Content: ovsServiceContent.Dropins[0].Contents,
+		}
+	}
+
+	return nil
+}
+
+func handleServices() (*k8sUpdateTarget, error) {
+	updateTarget := &k8sUpdateTarget{}
+
+	// Check switchdev service
+	swdService, err := serviceManager.ReadService(switchdevService.Path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		// service not exists
+		updateTarget.switchdevService = true
+	} else {
+		needChange, err := service.CompareServices(swdService, switchdevService)
+		if err != nil {
+			return nil, err
+		}
+		updateTarget.switchdevService = needChange
+	}
+
+	// Check switchdev run script
+	data, err := ioutil.ReadFile(path.Join(chroot, switchdevRunScript.Path))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		updateTarget.switchdevScript = true
+	} else if string(data) != switchdevRunScript.Contents.Inline {
+		updateTarget.switchdevScript = true
+	}
+
+	// Check Network Manager
+	needChange, err := checkServiceNeedUpdate(networkManagerService)
+	if err != nil {
+		return nil, err
+	}
+	updateTarget.networkManager = needChange
+
+	// Check OpenVSwitch
+	needChange, err = checkServiceNeedUpdate(openVSwitchService)
+	if err != nil {
+		return nil, err
+	}
+	updateTarget.openVSwitch = needChange
+
+	return updateTarget, nil
+}
+
+func checkServiceNeedUpdate(serviceObj *service.Service) (bool, error) {
+	systemService, err := serviceManager.ReadService(serviceObj.Path)
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	if systemService != nil {
+		needChange, err := service.CompareServices(systemService, serviceObj)
+		if err != nil {
+			return false, err
+		}
+		return needChange, nil
+	}
+
+	return false, nil
+}
+
+func updateService(serviceObj *service.Service) error {
+	systemService, err := serviceManager.ReadService(serviceObj.Path)
+	if err != nil {
+		return err
+	}
+	if systemService == nil {
+		// Invalid case to reach here
+		return fmt.Errorf("k8s-plugin Apply(): can't update non-existing service %q", serviceObj.Name)
+	}
+	serviceOptions, err := unit.Deserialize(strings.NewReader(serviceObj.Content))
+	if err != nil {
+		return err
+	}
+	updatedService, err := service.AppendToService(systemService, serviceOptions...)
+	if err != nil {
+		return err
+	}
+
+	return serviceManager.EnableService(updatedService)
+}