diff --git a/Makefile b/Makefile index 0a3ecc370f..d44c55865e 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index ddaca085d6..31ac9efee2 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -50,6 +50,13 @@ spec: volumeMounts: - name: host mountPath: /host + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "touch /host/etc/sriov-netowrk-operator-run"] + preStop: + exec: + command: ["/bin/sh","-c","rm -f /host//etc/sriov-netowrk-operator-run"] volumes: - name: host hostPath: diff --git a/bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml b/bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml index a0c3bf4ddf..dcac947b29 100644 --- a/bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml +++ b/bindata/manifests/machine-config/switchdev-units/switchdev-configuration.yaml @@ -2,7 +2,7 @@ contents: | [Unit] Description=Configures SRIOV NIC into switchdev mode # Removal of this file signals firstboot completion - ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json + ConditionPathExists=/etc/sriov-netowrk-operator-run # This service is used to move a SRIOV NIC into switchdev mode Wants=network-pre.target Before=network-pre.target diff --git a/go.mod b/go.mod index d01a627608..08df963f31 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/yaml.v2 v2.3.0 k8s.io/api v0.19.0 k8s.io/apimachinery v0.19.0 k8s.io/client-go v0.19.0 diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 12facee248..f958c6f5d7 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = registerPlugins(ns) if utils.ClusterType == utils.ClusterTypeOpenshift { pl = append(pl, McoPlugin) + } else { + pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) } diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index dd59de90fe..ffab8607d9 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -31,6 +31,7 @@ const ( GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" McoPlugin = "mco_plugin" + K8sPlugin = "k8s_plugin" ) // loadPlugin loads a single plugin from a file path diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go new file mode 100644 index 0000000000..08b5b63940 --- /dev/null +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -0,0 +1,269 @@ +package main + +import ( + "io/ioutil" + "path" + "strings" + + "github.com/golang/glog" + "gopkg.in/yaml.v2" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type K8sPlugin struct { + PluginName string + SpecVersion string +} + +type McoService struct { + Contents string +} + +type McoFile struct { + Path string + Contents struct { + Inline string + } +} + +type McoServiceInjection struct { + Dropins []struct { + Contents string + } +} + +const ( + mcoManifestPath = "bindata/manifests/machine-config/" + switchdevUnits = mcoManifestPath + "switchdev-units/" + switchdevUnitFile = switchdevUnits + "switchdev-configuration.yaml" + networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" + configuresSwitchdevScript = mcoManifestPath + "files/configure-switchdev.sh.yaml" + ovsVSwitchdUnitFile = mcoManifestPath + "ovs-units/ovs-vswitchd.service.yaml" + + switchdevService = "switchdev-configuration" + networkManagerService = "NetworkManager" + + switchdevServicePath = "/etc/systemd/system/" + switchdevService + ".service" + systemdDir = "/usr/lib/systemd/system" +) + +var ( + Plugin K8sPlugin + + serviceManager service.ServiceManager + + ovsServiceName = "openvswitch" + + needServices bool + needInjectServices bool +) + +// Initialize our plugin and set up initial values +func init() { + Plugin = K8sPlugin{ + PluginName: "k8s_plugin", + SpecVersion: "1.0", + } + + // ovs service name is different on ubuntu + serviceManager = service.NewServiceManager("/host") + data, err := ioutil.ReadFile("/host/etc/os-release") + if err != nil { + panic(err) + } + + osInfo := string(data) + if strings.Contains(osInfo, "ubuntu") { + ovsServiceName = "openvswitch-switch" + } +} + +// Name returns the name of the plugin +func (p *K8sPlugin) Name() string { + return p.PluginName +} + +// Spec returns the version of the spec expected by the plugin +func (p *K8sPlugin) Spec() string { + return p.SpecVersion +} + +// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateAdd()") + return p.OnNodeStateChange(nil, state) +} + +// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateChange()") + needDrain = false + needReboot = false + + // Check services + needServices, err = handleServices() + if err != nil { + return + } + + if !needServices { + needServices, err = serviceManager.CheckServicesLoaded() + if err != nil { + return + } + } + needDrain = needServices + + needInjectServices, err = handleServiceInjections() + if err != nil { + return false, false, err + } + if !needInjectServices { + needInjectServices, err = serviceManager.CheckServiceInjections() + } + needDrain = needDrain || needInjectServices + + // Check switchdev config + var update, remove bool + if new == nil { + return + } + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + return + } + if remove { + glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") + needDrain = true + needReboot = true + return + } + if update { + glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") + needDrain = true + needReboot = true + return + } + return +} + +// Apply config change +func (p *K8sPlugin) Apply() error { + glog.Info("k8s-plugin Apply()") + if needServices { + if err := serviceManager.LoadMissingServices(); err != nil { + return err + } + needServices = false + } + + if needInjectServices { + if err := serviceManager.InjectServices(); err != nil { + return err + } + needInjectServices = false + } + + return nil +} + +func handleServices() (bool, error) { + needChange := false + smServices := serviceManager.ServicesSet() + // Check switchdev service + if _, ok := smServices[switchdevService]; !ok { + needChange = true + data, err := ioutil.ReadFile(switchdevUnitFile) + if err != nil { + return false, err + } + + var switchdevServiceContent McoService + if err := yaml.Unmarshal(data, &switchdevServiceContent); err != nil { + return false, err + } + + data, err = ioutil.ReadFile(configuresSwitchdevScript) + if err != nil { + return false, err + } + var switchdevRunFile McoFile + if err := yaml.Unmarshal(data, &switchdevRunFile); err != nil { + return false, err + } + + switchdevService := &service.Service{ + Name: switchdevService, + Path: switchdevServicePath, + Body: switchdevServiceContent.Contents, + Run: switchdevRunFile.Path, + RunBody: switchdevRunFile.Contents.Inline, + } + + if err = serviceManager.AddService(switchdevService); err != nil { + return false, err + } + } + + return needChange, nil +} + +func handleServiceInjections() (bool, error) { + needChange := false + smServices := serviceManager.InjectableServicesSet() + + // Check ovs service exists + if _, ok := smServices[ovsServiceName]; !ok { + needChange = true + + data, err := ioutil.ReadFile(ovsVSwitchdUnitFile) + if err != nil { + return false, err + } + var ovsServiceContent McoServiceInjection + if err := yaml.Unmarshal(data, &ovsServiceContent); err != nil { + return false, err + } + + ovsService, err := service.NewInjectableService(ovsServiceName, + path.Join(systemdDir, ovsServiceName+".service"), + ovsServiceContent.Dropins[0].Contents) + if err != nil { + return false, err + } + + if err = serviceManager.AddInjectableService(ovsService); err != nil { + return false, err + } + } + + // Check NetworkManager service exists + if _, ok := smServices[networkManagerService]; !ok { + needChange = true + + data, err := ioutil.ReadFile(networkManagerUnitFile) + if err != nil { + return false, err + } + var networkManagerServiceContent McoServiceInjection + if err := yaml.Unmarshal(data, &networkManagerServiceContent); err != nil { + return false, err + } + + networkMgrService, err := service.NewInjectableService(networkManagerService, + path.Join(systemdDir, networkManagerService+".service"), + networkManagerServiceContent.Dropins[0].Contents) + if err != nil { + return false, err + } + + if err = serviceManager.AddInjectableService(networkMgrService); err != nil { + return false, err + } + } + + return needChange, nil +} diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go index 215ae7fd80..74f3fb4c22 100644 --- a/pkg/plugins/mco/mco_plugin.go +++ b/pkg/plugins/mco/mco_plugin.go @@ -3,8 +3,6 @@ package main import ( "context" "encoding/json" - "fmt" - "io/ioutil" "os" "github.com/golang/glog" @@ -17,6 +15,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) type McoPlugin struct { @@ -28,7 +27,6 @@ type McoPlugin struct { const ( switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - switchDevConfPath = "/host/etc/switchdev.conf" nodeLabelPrefix = "node-role.kubernetes.io/" ) @@ -89,7 +87,7 @@ func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS } var update, remove bool - if update, remove, err = writeSwitchdevConfFile(new); err != nil { + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) return } @@ -156,46 +154,3 @@ func (p *McoPlugin) Apply() error { glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) return nil } - -func writeSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { - _, err = os.Stat(switchDevConfPath) - if err != nil { - if os.IsNotExist(err) { - glog.V(2).Infof("writeSwitchdevConfFile(): file not existed, create it") - _, err = os.Create(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to create file: %v", err) - return - } - } else { - return - } - } - newContent := "" - for _, iface := range newState.Spec.Interfaces { - if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { - newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) - } - } - oldContent, err := ioutil.ReadFile(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to read file: %v", err) - return - } - if newContent == string(oldContent) { - glog.V(2).Info("writeSwitchdevConfFile(): no update") - return - } - if newContent == "" { - remove = true - glog.V(2).Info("writeSwitchdevConfFile(): remove content in switchdev.conf") - } - update = true - glog.V(2).Infof("writeSwitchdevConfFile(): write %s to switchdev.conf", newContent) - err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to write file: %v", err) - return - } - return -} diff --git a/pkg/service/InjectableService.go b/pkg/service/InjectableService.go new file mode 100644 index 0000000000..d830e3e6c2 --- /dev/null +++ b/pkg/service/InjectableService.go @@ -0,0 +1,25 @@ +package service + +import ( + "github.com/coreos/go-systemd/v22/unit" + "strings" +) + +type InjectableService struct { + Name string + Path string + options []*unit.UnitOption +} + +func NewInjectableService(name, path, body string) (*InjectableService, error) { + opts, err := unit.Deserialize(strings.NewReader(body)) + if err != nil { + return nil, err + } + + return &InjectableService{ + Name: name, + Path: path, + options: opts, + }, nil +} diff --git a/pkg/service/Service.go b/pkg/service/Service.go new file mode 100644 index 0000000000..c994090be3 --- /dev/null +++ b/pkg/service/Service.go @@ -0,0 +1,27 @@ +package service + +import ( + "io/ioutil" +) + +type Service struct { + Name string + Path string + Body string + Run string + RunBody string +} + +func ServiceFromFile(name, file string) (*Service, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + service := &Service{ + Name: name, + Body: string(data), + } + + return service, nil +} diff --git a/pkg/service/serviceManager.go b/pkg/service/serviceManager.go new file mode 100644 index 0000000000..40d60de449 --- /dev/null +++ b/pkg/service/serviceManager.go @@ -0,0 +1,234 @@ +package service + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + + "github.com/coreos/go-systemd/v22/unit" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type ServiceManager interface { + AddService(*Service) error + AddInjectableService(*InjectableService) error + LoadMissingServices() error + CheckServicesLoaded() (bool, error) + ServicesSet() map[string]bool + InjectableServicesSet() map[string]bool + CheckServiceInjections() (bool, error) + InjectServices() error +} + +type serviceManager struct { + chroot string + services map[string]*Service + injectableServices map[string]*InjectableService +} + +func NewServiceManager(chroot string) ServiceManager { + return &serviceManager{ + chroot: chroot, + services: map[string]*Service{}, + injectableServices: map[string]*InjectableService{}} +} + +func (sm *serviceManager) AddService(service *Service) error { + if _, ok := sm.services[service.Name]; ok { + return fmt.Errorf("serviceManager AddService(): service %s already exists", service.Name) + } + + sm.services[service.Name] = service + return nil +} + +func (sm *serviceManager) AddInjectableService(service *InjectableService) error { + if _, ok := sm.injectableServices[service.Name]; ok { + return fmt.Errorf("serviceManager AddInjectableService(): service %s already exists", service.Name) + } + + sm.injectableServices[service.Name] = service + return nil +} + +func (sm *serviceManager) loadServiceOnHost(service *Service) error { + // Write service file + err := ioutil.WriteFile(path.Join(sm.chroot, service.Path), []byte(service.Body), 0644) + if err != nil { + return err + } + + if service.Run != "" && service.RunBody != "" { + // Write service run + err = ioutil.WriteFile(path.Join(sm.chroot, service.Run), []byte(service.RunBody), 0755) + if err != nil { + return err + } + } + + // Change root dir + exit, err := utils.Chroot(sm.chroot) + if err != nil { + return fmt.Errorf("loadServiceOnHost(): service %q, failed with: %v", service.Name, err) + } + defer exit() + + // Enable service + cmd := exec.Command("systemctl", "enable", service.Name) + return cmd.Run() +} + +func (sm *serviceManager) isServiceLoaded(servicePath string) (bool, error) { + _, err := os.Stat(path.Join(sm.chroot, servicePath)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +func (sm *serviceManager) CheckServicesLoaded() (bool, error) { + for _, service := range sm.services { + exists, err := sm.isServiceLoaded(service.Path) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + } + + return true, nil +} + +func (sm *serviceManager) LoadMissingServices() error { + for _, service := range sm.services { + exists, err := sm.isServiceLoaded(service.Path) + if err != nil { + return err + } + if exists { + continue + } + if err := sm.loadServiceOnHost(service); err != nil { + return err + } + } + + return nil +} + +func (sm *serviceManager) ServicesSet() map[string]bool { + services := map[string]bool{} + for service := range sm.services { + services[service] = true + } + + return services +} + +func (sm *serviceManager) InjectableServicesSet() map[string]bool { + services := map[string]bool{} + for service := range sm.injectableServices { + services[service] = true + } + + return services +} + +func (sm *serviceManager) CheckServiceInjections() (bool, error) { + for _, service := range sm.injectableServices { + exists, err := sm.isServiceLoaded(service.Path) + if err != nil { + return false, err + } + if !exists { + // Skip non existing services + continue + } + + data, err := ioutil.ReadFile(path.Join(sm.chroot, service.Path)) + if err != nil { + return false, err + } + opts, err := unit.Deserialize(strings.NewReader(string(data))) + if err != nil { + return false, err + } + + OUTER: + for _, serviceOpt := range service.options { + for _, opt := range opts { + if serviceOpt.Match(opt) { + continue OUTER + } + } + + return true, nil + } + } + + return false, nil +} + +func (sm *serviceManager) InjectServices() error { + for _, service := range sm.injectableServices { + exists, err := sm.isServiceLoaded(service.Path) + if err != nil { + return err + } + if !exists { + // Skip non existing services + continue + } + + data, err := ioutil.ReadFile(path.Join(sm.chroot, service.Path)) + if err != nil { + return err + } + opts, err := unit.Deserialize(strings.NewReader(string(data))) + if err != nil { + return err + } + + OUTER: + for _, serviceOpt := range service.options { + for _, opt := range opts { + if serviceOpt.Match(opt) { + continue OUTER + } + } + + // Update service with new option + opts = append(opts, serviceOpt) + } + + data, err = ioutil.ReadAll(unit.Serialize(opts)) + if err != nil { + return err + } + + err = ioutil.WriteFile(path.Join(sm.chroot, service.Path), data, 0644) + if err != nil { + return err + } + } + + // Change root dir + exit, err := utils.Chroot(sm.chroot) + if err != nil { + return err + } + defer exit() + + // Reload services + cmd := exec.Command("systemctl", "daemon-reload") + return cmd.Run() +} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go new file mode 100644 index 0000000000..818b53de2c --- /dev/null +++ b/pkg/utils/switchdev.go @@ -0,0 +1,58 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/golang/glog" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +const ( + switchDevConfPath = "/host/etc/switchdev.conf" +) + +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { + _, err = os.Stat(switchDevConfPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("WriteSwitchdevConfFile(): file not existed, create it") + _, err = os.Create(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to create file: %v", err) + return + } + } else { + return + } + } + newContent := "" + for _, iface := range newState.Spec.Interfaces { + if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { + newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) + } + } + oldContent, err := ioutil.ReadFile(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to read file: %v", err) + return + } + if newContent == string(oldContent) { + glog.V(2).Info("WriteSwitchdevConfFile(): no update") + return + } + if newContent == "" { + remove = true + glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") + } + update = true + glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) + err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to write file: %v", err) + return + } + return +}