From 5c079545bcfe5ca9b4125f82340d6d09f4ef71a4 Mon Sep 17 00:00:00 2001 From: Mamduh Alassi Date: Thu, 7 Jan 2021 08:15:41 +0200 Subject: [PATCH] Support changing Eswitch mode of SR-IOV NICs for kubernetes deployment Signed-off-by: Mamduh Alassi --- Makefile | 2 +- bindata/manifests/daemon/daemonset.yaml | 4 + .../files/configure-switchdev.sh.yaml | 0 .../machineconfigpool.yaml | 0 .../ovs-units/ovs-vswitchd.service.yaml | 0 .../NetworkManager.service.yaml | 0 .../switchdev-configuration.yaml | 0 bindata/scripts/clean-k8s-services.sh | 30 ++ controllers/sriovoperatorconfig_controller.go | 4 +- go.mod | 2 + pkg/daemon/daemon.go | 2 + pkg/daemon/plugin.go | 1 + pkg/plugins/k8s/k8s_plugin.go | 354 ++++++++++++++++++ pkg/plugins/mco/mco_plugin.go | 49 +-- pkg/service/service.go | 15 + pkg/service/service_manager.go | 76 ++++ pkg/service/types.go | 23 ++ pkg/service/utils.go | 149 ++++++++ pkg/utils/switchdev.go | 58 +++ 19 files changed, 719 insertions(+), 50 deletions(-) rename bindata/manifests/{machine-config => switchdev-config}/files/configure-switchdev.sh.yaml (100%) rename bindata/manifests/{machine-config => switchdev-config}/machineconfigpool.yaml (100%) rename bindata/manifests/{machine-config => switchdev-config}/ovs-units/ovs-vswitchd.service.yaml (100%) rename bindata/manifests/{machine-config => switchdev-config}/switchdev-units/NetworkManager.service.yaml (100%) rename bindata/manifests/{machine-config => switchdev-config}/switchdev-units/switchdev-configuration.yaml (100%) create mode 100755 bindata/scripts/clean-k8s-services.sh create mode 100644 pkg/plugins/k8s/k8s_plugin.go create mode 100644 pkg/service/service.go create mode 100644 pkg/service/service_manager.go create mode 100644 pkg/service/types.go create mode 100644 pkg/service/utils.go create mode 100644 pkg/utils/switchdev.go diff --git a/Makefile b/Makefile index 0a3ecc370f..d44c55865e 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index ddaca085d6..aa6edb095b 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -50,6 +50,10 @@ spec: volumeMounts: - name: host mountPath: /host + lifecycle: + preStop: + exec: + command: ["/bindata/scripts/clean-k8s-services.sh"] volumes: - name: host hostPath: diff --git a/bindata/manifests/machine-config/files/configure-switchdev.sh.yaml b/bindata/manifests/switchdev-config/files/configure-switchdev.sh.yaml similarity index 100% rename from bindata/manifests/machine-config/files/configure-switchdev.sh.yaml rename to bindata/manifests/switchdev-config/files/configure-switchdev.sh.yaml diff --git a/bindata/manifests/machine-config/machineconfigpool.yaml b/bindata/manifests/switchdev-config/machineconfigpool.yaml similarity index 100% rename from bindata/manifests/machine-config/machineconfigpool.yaml rename to bindata/manifests/switchdev-config/machineconfigpool.yaml diff --git a/bindata/manifests/machine-config/ovs-units/ovs-vswitchd.service.yaml b/bindata/manifests/switchdev-config/ovs-units/ovs-vswitchd.service.yaml similarity index 100% rename from bindata/manifests/machine-config/ovs-units/ovs-vswitchd.service.yaml rename to bindata/manifests/switchdev-config/ovs-units/ovs-vswitchd.service.yaml diff --git a/bindata/manifests/machine-config/switchdev-units/NetworkManager.service.yaml b/bindata/manifests/switchdev-config/switchdev-units/NetworkManager.service.yaml similarity index 100% rename from bindata/manifests/machine-config/switchdev-units/NetworkManager.service.yaml rename to bindata/manifests/switchdev-config/switchdev-units/NetworkManager.service.yaml diff --git a/bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml b/bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration.yaml similarity index 100% rename from bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml rename to bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration.yaml diff --git a/bindata/scripts/clean-k8s-services.sh b/bindata/scripts/clean-k8s-services.sh new file mode 100755 index 0000000000..cc85552067 --- /dev/null +++ b/bindata/scripts/clean-k8s-services.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +chroot_path="/host" + +function clean_services() { + # Remove switchdev service files + rm -f $chroot_path/etc/systemd/system/switchdev-configuration.service + rm -f $chroot_path/usr/local/bin/configure-switchdev.sh + rm -f $chroot_path/etc/switchdev.conf + + # clean NetworkManager and ovs-vswitchd services + network_manager_service=$chroot_path/usr/lib/systemd/system/NetworkManager.service + ovs_service=$chroot_path/usr/lib/systemd/system/ovs-vswitchd.service + + if [ -f $network_manager_service ]; then + sed -i.bak '/switchdev-configuration.service/d' $network_manager_service + fi + + if [ -f $ovs_service ]; then + sed -i.bak '/hw-offload/d' $ovs_service + fi +} + +clean_services +# Reload host services +chroot $chroot_path /bin/bash -c systemctl daemon-reload >/dev/null 2>&1 || true + +# Restart system services +chroot $chroot_path /bin/bash -c systemctl restart NetworkManager.service >/dev/null 2>&1 || true +chroot $chroot_path /bin/bash -c systemctl restart ovs-vswitchd.service >/dev/null 2>&1 || true diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index ea40d87d16..5a191c7181 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -417,11 +417,11 @@ func (r *SriovOperatorConfigReconciler) syncOffloadMachineConfig(dc *sriovnetwor data.Data["HwOffloadNodeLabel"] = HwOffloadNodeLabel mcName := "00-" + HwOffloadNodeLabel mcpName := HwOffloadNodeLabel - mc, err := render.GenerateMachineConfig("bindata/manifests/machine-config", mcName, HwOffloadNodeLabel, dc.Spec.EnableOvsOffload, &data) + mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, HwOffloadNodeLabel, dc.Spec.EnableOvsOffload, &data) if err != nil { return err } - mcpRaw, err := render.RenderTemplate("bindata/manifests/machine-config/machineconfigpool.yaml", &data) + mcpRaw, err := render.RenderTemplate("bindata/manifests/switchdev-config/machineconfigpool.yaml", &data) if err != nil { return err } diff --git a/go.mod b/go.mod index d01a627608..bd41331e3b 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/blang/semver v3.5.0+incompatible github.com/cenkalti/backoff v2.2.1+incompatible + github.com/coreos/go-systemd/v22 v22.0.0 github.com/fsnotify/fsnotify v1.4.9 github.com/go-logr/logr v0.2.1 github.com/go-logr/zapr v0.2.0 // indirect @@ -29,6 +30,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/yaml.v2 v2.3.0 k8s.io/api v0.19.0 k8s.io/apimachinery v0.19.0 k8s.io/client-go v0.19.0 diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 12facee248..f958c6f5d7 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = registerPlugins(ns) if utils.ClusterType == utils.ClusterTypeOpenshift { pl = append(pl, McoPlugin) + } else { + pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) } diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index dd59de90fe..ffab8607d9 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -31,6 +31,7 @@ const ( GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" McoPlugin = "mco_plugin" + K8sPlugin = "k8s_plugin" ) // loadPlugin loads a single plugin from a file path diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go new file mode 100644 index 0000000000..5fc7a78e28 --- /dev/null +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -0,0 +1,354 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "strings" + + "github.com/coreos/go-systemd/v22/unit" + "github.com/golang/glog" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type K8sPlugin struct { + PluginName string + SpecVersion string + serviceManager service.ServiceManager + switchdevRunScript *service.ScriptManifestFile + switchdevService *service.Service + openVSwitchService *service.Service + networkManagerService *service.Service + updateTarget *k8sUpdateTarget +} + +type k8sUpdateTarget struct { + switchdevService bool + switchdevScript bool + systemServices []*service.Service +} + +func (u *k8sUpdateTarget) needUpdate() bool { + return u.switchdevService || u.switchdevScript || len(u.systemServices) > 0 +} + +func (u *k8sUpdateTarget) reset() { + u.switchdevService = false + u.switchdevScript = false + u.systemServices = []*service.Service{} +} + +func (u *k8sUpdateTarget) String() string { + var updateList []string + if u.switchdevService { + updateList = append(updateList, "SwitchdevService") + } + if u.switchdevScript { + updateList = append(updateList, "SwitchdevScript") + } + for _, s := range u.systemServices { + updateList = append(updateList, s.Name) + } + + return strings.Join(updateList, ",") +} + +const ( + switchdevManifestPath = "bindata/manifests/switchdev-config/" + switchdevUnits = switchdevManifestPath + "switchdev-units/" + switchdevUnitFile = switchdevUnits + "switchdev-configuration.yaml" + networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" + ovsUnitFile = switchdevManifestPath + "ovs-units/ovs-vswitchd.service.yaml" + configuresSwitchdevScript = switchdevManifestPath + "files/configure-switchdev.sh.yaml" + + chroot = "/host" +) + +var ( + Plugin K8sPlugin +) + +// Initialize our plugin and set up initial values +func init() { + Plugin = K8sPlugin{ + PluginName: "k8s_plugin", + SpecVersion: "1.0", + serviceManager: service.NewServiceManager(chroot), + updateTarget: &k8sUpdateTarget{}, + } + + // Read manifest files for plugin + if err := Plugin.readManifestFiles(); err != nil { + panic(err) + } +} + +// Name returns the name of the plugin +func (p *K8sPlugin) Name() string { + return p.PluginName +} + +// Spec returns the version of the spec expected by the plugin +func (p *K8sPlugin) Spec() string { + return p.SpecVersion +} + +// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateAdd()") + return p.OnNodeStateChange(nil, state) +} + +// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateChange()") + needDrain = false + needReboot = false + + p.updateTarget.reset() + // Check services + err = p.servicesStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) + return + } + + // Check switchdev config + var update, remove bool + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + return + } + if remove { + glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") + needDrain = true + needReboot = true + return + } + if update { + glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") + needDrain = true + needReboot = true + return + } + if p.updateTarget.needUpdate() { + glog.Infof("k8s-plugin OnNodeStateChange(): needDrain to update %q", p.updateTarget) + needDrain = true + } + + return +} + +// Apply config change +func (p *K8sPlugin) Apply() error { + glog.Info("k8s-plugin Apply()") + if err := p.updateSwichdevService(); err != nil { + return err + } + + for _, systemService := range p.updateTarget.systemServices { + if err := p.updateSystemService(systemService); err != nil { + return err + } + } + + return nil +} + +func (p *K8sPlugin) readSwitchdevManifest() error { + switchdevService, err := service.ReadServiceManifestFile(switchdevUnitFile) + if err != nil { + return err + } + + // Remove run condition form the service + conditionOpt := &unit.UnitOption{ + Section: "Unit", + Name: "ConditionPathExists", + Value: "!/etc/ignition-machine-config-encapsulated.json", + } + switchdevService, err = service.RemoveFromService(switchdevService, conditionOpt) + if err != nil { + return err + } + p.switchdevService = switchdevService + + switchdevRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevScript) + if err != nil { + return err + } + + p.switchdevRunScript = switchdevRunScript + + return nil +} + +func (p *K8sPlugin) readNetworkManagerManifest() error { + networkManagerService, err := service.ReadServiceInjectionManifestFile(networkManagerUnitFile) + if err != nil { + return err + } + + p.networkManagerService = networkManagerService + return nil +} + +func (p *K8sPlugin) readOpenVSwitchdManifest() error { + openVSwitchService, err := service.ReadServiceInjectionManifestFile(ovsUnitFile) + if err != nil { + return err + } + + p.openVSwitchService = openVSwitchService + return nil +} + +func (p *K8sPlugin) readManifestFiles() error { + if err := p.readSwitchdevManifest(); err != nil { + return err + } + + if err := p.readNetworkManagerManifest(); err != nil { + return err + } + + if err := p.readOpenVSwitchdManifest(); err != nil { + return err + } + + return nil +} + +func (p *K8sPlugin) switchdevServiceStateUpdate() error { + // Check switchdev service + swdService, err := p.serviceManager.ReadService(p.switchdevService.Path) + if err != nil { + if !os.IsNotExist(err) { + return err + } + // service not exists + p.updateTarget.switchdevService = true + } else { + needChange, err := service.CompareServices(swdService, p.switchdevService) + if err != nil { + return err + } + p.updateTarget.switchdevService = needChange + } + + // Check switchdev run script + data, err := ioutil.ReadFile(path.Join(chroot, p.switchdevRunScript.Path)) + if err != nil { + if !os.IsNotExist(err) { + return err + } + p.updateTarget.switchdevScript = true + } else if string(data) != p.switchdevRunScript.Contents.Inline { + p.updateTarget.switchdevScript = true + } + + return nil +} + +func (p *K8sPlugin) getSystemServices() []*service.Service { + return []*service.Service{p.networkManagerService, p.openVSwitchService} +} + +func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool { + systemService, err := p.serviceManager.ReadService(serviceObj.Path) + if err != nil { + glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to read switchdev service file %q: %v", + serviceObj.Path, err) + return false + } + if systemService != nil { + needChange, err := service.CompareServices(systemService, serviceObj) + if err != nil { + glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to compare switchdev service : %v", err) + return false + } + return needChange + } + + return false +} + +func (p *K8sPlugin) systemServicesStateUpdate() error { + var services []*service.Service + for _, systemService := range p.getSystemServices() { + exist, err := p.serviceManager.IsServiceExist(systemService.Path) + if err != nil { + return err + } + if !exist { + glog.Errorf("k8s-plugin systemServicesStateUpdate(): %q not found", systemService.Name) + return err + } + if p.isSystemServiceNeedUpdate(systemService) { + services = append(services, systemService) + } + } + + p.updateTarget.systemServices = services + return nil +} + +func (p *K8sPlugin) servicesStateUpdate() error { + // Check switchdev + err := p.switchdevServiceStateUpdate() + if err != nil { + return err + } + + // Check system services + err = p.systemServicesStateUpdate() + if err != nil { + return err + } + + return nil +} + +func (p *K8sPlugin) updateSwichdevService() error { + if p.updateTarget.switchdevService { + err := p.serviceManager.EnableService(p.switchdevService) + if err != nil { + return err + } + } + + if p.updateTarget.switchdevScript { + err := ioutil.WriteFile(path.Join(chroot, p.switchdevRunScript.Path), + []byte(p.switchdevRunScript.Contents.Inline), 0755) + if err != nil { + return err + } + } + + return nil +} + +func (p *K8sPlugin) updateSystemService(serviceObj *service.Service) error { + systemService, err := p.serviceManager.ReadService(serviceObj.Path) + if err != nil { + return err + } + if systemService == nil { + // Invalid case to reach here + return fmt.Errorf("k8s-plugin Apply(): can't update non-existing service %q", serviceObj.Name) + } + serviceOptions, err := unit.Deserialize(strings.NewReader(serviceObj.Content)) + if err != nil { + return err + } + updatedService, err := service.AppendToService(systemService, serviceOptions...) + if err != nil { + return err + } + + return p.serviceManager.EnableService(updatedService) +} diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go index 215ae7fd80..74f3fb4c22 100644 --- a/pkg/plugins/mco/mco_plugin.go +++ b/pkg/plugins/mco/mco_plugin.go @@ -3,8 +3,6 @@ package main import ( "context" "encoding/json" - "fmt" - "io/ioutil" "os" "github.com/golang/glog" @@ -17,6 +15,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) type McoPlugin struct { @@ -28,7 +27,6 @@ type McoPlugin struct { const ( switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - switchDevConfPath = "/host/etc/switchdev.conf" nodeLabelPrefix = "node-role.kubernetes.io/" ) @@ -89,7 +87,7 @@ func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS } var update, remove bool - if update, remove, err = writeSwitchdevConfFile(new); err != nil { + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) return } @@ -156,46 +154,3 @@ func (p *McoPlugin) Apply() error { glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) return nil } - -func writeSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { - _, err = os.Stat(switchDevConfPath) - if err != nil { - if os.IsNotExist(err) { - glog.V(2).Infof("writeSwitchdevConfFile(): file not existed, create it") - _, err = os.Create(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to create file: %v", err) - return - } - } else { - return - } - } - newContent := "" - for _, iface := range newState.Spec.Interfaces { - if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { - newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) - } - } - oldContent, err := ioutil.ReadFile(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to read file: %v", err) - return - } - if newContent == string(oldContent) { - glog.V(2).Info("writeSwitchdevConfFile(): no update") - return - } - if newContent == "" { - remove = true - glog.V(2).Info("writeSwitchdevConfFile(): remove content in switchdev.conf") - } - update = true - glog.V(2).Infof("writeSwitchdevConfFile(): write %s to switchdev.conf", newContent) - err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to write file: %v", err) - return - } - return -} diff --git a/pkg/service/service.go b/pkg/service/service.go new file mode 100644 index 0000000000..671d2e20af --- /dev/null +++ b/pkg/service/service.go @@ -0,0 +1,15 @@ +package service + +type Service struct { + Name string + Path string + Content string +} + +func NewService(name, path, content string) *Service { + return &Service{ + Name: name, + Path: path, + Content: content, + } +} diff --git a/pkg/service/service_manager.go b/pkg/service/service_manager.go new file mode 100644 index 0000000000..ddd9a1f4ed --- /dev/null +++ b/pkg/service/service_manager.go @@ -0,0 +1,76 @@ +package service + +import ( + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type ServiceManager interface { + IsServiceExist(string) (bool, error) + ReadService(string) (*Service, error) + EnableService(service *Service) error +} + +type serviceManager struct { + chroot string +} + +func NewServiceManager(chroot string) ServiceManager { + root := chroot + if root == "" { + root = "/" + } + return &serviceManager{root} +} + +// ReadService read service from given path +func (sm *serviceManager) IsServiceExist(servicePath string) (bool, error) { + _, err := os.Stat(path.Join(sm.chroot, servicePath)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +// ReadService read service from given path +func (sm *serviceManager) ReadService(servicePath string) (*Service, error) { + data, err := ioutil.ReadFile(path.Join(sm.chroot, servicePath)) + if err != nil { + return nil, err + } + + return &Service{ + Name: filepath.Base(servicePath), + Path: servicePath, + Content: string(data), + }, nil +} + +// EnableService creates service file and enables it with systemctl enable +func (sm *serviceManager) EnableService(service *Service) error { + // Write service file + err := ioutil.WriteFile(path.Join(sm.chroot, service.Path), []byte(service.Content), 0644) + if err != nil { + return err + } + + // Change root dir + exit, err := utils.Chroot(sm.chroot) + if err != nil { + return err + } + defer exit() + + // Enable service + cmd := exec.Command("systemctl", "enable", service.Name) + return cmd.Run() +} diff --git a/pkg/service/types.go b/pkg/service/types.go new file mode 100644 index 0000000000..aaa879a4d8 --- /dev/null +++ b/pkg/service/types.go @@ -0,0 +1,23 @@ +package service + +// ServiceInjectionManifestFile service injection manifest file structure +type ServiceInjectionManifestFile struct { + Name string + Dropins []struct { + Contents string + } +} + +// ServiceManifestFile service manifest file structure +type ServiceManifestFile struct { + Name string + Contents string +} + +// ScriptManifestFile script manifest file structure +type ScriptManifestFile struct { + Path string + Contents struct { + Inline string + } +} diff --git a/pkg/service/utils.go b/pkg/service/utils.go new file mode 100644 index 0000000000..8b9e82c723 --- /dev/null +++ b/pkg/service/utils.go @@ -0,0 +1,149 @@ +package service + +import ( + "io/ioutil" + "strings" + + "github.com/coreos/go-systemd/v22/unit" + "gopkg.in/yaml.v2" +) + +const systemdDir = "/usr/lib/systemd/system/" + +// CompareServices compare 2 service and return true if serviceA has all the fields of serviceB +func CompareServices(serviceA, serviceB *Service) (bool, error) { + optsA, err := unit.Deserialize(strings.NewReader(serviceA.Content)) + if err != nil { + return false, err + } + optsB, err := unit.Deserialize(strings.NewReader(serviceB.Content)) + if err != nil { + return false, err + } + +OUTER: + for _, optB := range optsB { + for _, optA := range optsA { + if optA.Match(optB) { + continue OUTER + } + } + + return true, nil + } + + return false, nil +} + +// RemoveFromService removes given fields from service +func RemoveFromService(service *Service, options ...*unit.UnitOption) (*Service, error) { + opts, err := unit.Deserialize(strings.NewReader(service.Content)) + if err != nil { + return nil, err + } + + var newServiceOptions []*unit.UnitOption +OUTER: + for _, opt := range opts { + for _, optRemove := range options { + if opt.Match(optRemove) { + continue OUTER + } + } + + newServiceOptions = append(newServiceOptions, opt) + } + + data, err := ioutil.ReadAll(unit.Serialize(newServiceOptions)) + if err != nil { + return nil, err + } + + return &Service{ + Name: service.Name, + Path: service.Path, + Content: string(data), + }, nil +} + +// AppendToService appends given fields to service +func AppendToService(service *Service, options ...*unit.UnitOption) (*Service, error) { + serviceOptions, err := unit.Deserialize(strings.NewReader(service.Content)) + if err != nil { + return nil, err + } + +OUTER: + for _, appendOpt := range options { + for _, opt := range serviceOptions { + if opt.Match(appendOpt) { + continue OUTER + } + } + serviceOptions = append(serviceOptions, appendOpt) + } + + data, err := ioutil.ReadAll(unit.Serialize(serviceOptions)) + if err != nil { + return nil, err + } + + return &Service{ + Name: service.Name, + Path: service.Path, + Content: string(data), + }, nil +} + +// ReadServiceInjectionManifestFile reads service injection file +func ReadServiceInjectionManifestFile(path string) (*Service, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var serviceContent ServiceInjectionManifestFile + if err := yaml.Unmarshal(data, &serviceContent); err != nil { + return nil, err + } + + return &Service{ + Name: serviceContent.Name, + Path: systemdDir + serviceContent.Name, + Content: serviceContent.Dropins[0].Contents, + }, nil +} + +// ReadServiceManifestFile reads service file +func ReadServiceManifestFile(path string) (*Service, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var serviceFile *ServiceManifestFile + if err := yaml.Unmarshal(data, &serviceFile); err != nil { + return nil, err + } + + return &Service{ + Name: serviceFile.Name, + Path: "/etc/systemd/system/" + serviceFile.Name, + Content: serviceFile.Contents, + }, nil +} + +// ReadScriptManifestFile reads script file +func ReadScriptManifestFile(path string) (*ScriptManifestFile, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var scriptFile *ScriptManifestFile + if err := yaml.Unmarshal(data, &scriptFile); err != nil { + return nil, err + } + + return scriptFile, nil +} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go new file mode 100644 index 0000000000..716fed5a5a --- /dev/null +++ b/pkg/utils/switchdev.go @@ -0,0 +1,58 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/golang/glog" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +const ( + switchDevConfPath = "/host/etc/switchdev.conf" +) + +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { + _, err = os.Stat(switchDevConfPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("WriteSwitchdevConfFile(): file not existed, create it") + _, err = os.Create(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to create file: %v", err) + return + } + } else { + return + } + } + newContent := "" + for _, iface := range newState.Spec.Interfaces { + if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { + newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) + } + } + oldContent, err := ioutil.ReadFile(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to read file: %v", err) + return + } + if newContent == string(oldContent) { + glog.V(2).Info("WriteSwitchdevConfFile(): no update") + return + } + if newContent == "" { + remove = true + glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") + } + update = true + glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) + err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0644) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to write file: %v", err) + return + } + return +}