diff --git a/Makefile b/Makefile index 0a3ecc370f..d44c55865e 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index ddaca085d6..31ac9efee2 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -50,6 +50,13 @@ spec: volumeMounts: - name: host mountPath: /host + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "touch /host/etc/sriov-netowrk-operator-run"] + preStop: + exec: + command: ["/bin/sh","-c","rm -f /host//etc/sriov-netowrk-operator-run"] volumes: - name: host hostPath: diff --git a/bindata/manifests/k8s-services/configure-switchdev.sh b/bindata/manifests/k8s-services/configure-switchdev.sh new file mode 100644 index 0000000000..6f83f5e90b --- /dev/null +++ b/bindata/manifests/k8s-services/configure-switchdev.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -eux +input="/etc/switchdev.conf" +if [ ! -f $input ]; then + echo "File /etc/switchdev.conf not exist." + exit +fi + +names=() + +while read pci_addr num_vfs; do + echo "Set $num_vfs VFs on device $pci_addr" + + names+=($(ls /sys/bus/pci/devices/${pci_addr}/net/)) + # create VFs + echo $num_vfs >/sys/bus/pci/devices/${pci_addr}/sriov_numvfs +done <"$input" + +# wait for vfs to be ready +sleep 5 +i=0 +while read pci_addr num_vfs; do + # unload VF driver + VfDirs=$(ls /sys/bus/pci/devices/${pci_addr} | grep virtfn) + for VfDir in $VfDirs; do + VfPciAddr=$(basename "$(readlink -f /sys/bus/pci/devices/${pci_addr}/$VfDir)") + echo $VfPciAddr >/sys/bus/pci/drivers/mlx5_core/unbind || true + + done + + # set PF to switchdev mode + devlink dev eswitch set pci/${pci_addr} mode switchdev + + # reset the pf name + new_name=$(ls /sys/bus/pci/devices/${pci_addr}/net/) + ip link set ${new_name} down + ip link set ${new_name} name ${names[i]} + ip link set ${names[i]} up + + # turn hw-tc-offload on + /usr/sbin/ethtool -K ${names[i]} hw-tc-offload on + + i=$((i + 1)) + + # load VF driver + for VfDir in $VfDirs; do + VfPciAddr=$(basename "$(readlink -f /sys/bus/pci/devices/${pci_addr}/$VfDir)") + echo $VfPciAddr >/sys/bus/pci/drivers_probe + done +done <"$input" diff --git a/bindata/manifests/k8s-services/ovs-vswitchd.service b/bindata/manifests/k8s-services/ovs-vswitchd.service new file mode 100644 index 0000000000..bf4193cca8 --- /dev/null +++ b/bindata/manifests/k8s-services/ovs-vswitchd.service @@ -0,0 +1,2 @@ +[Service] +ExecStart=/bin/ovs-vsctl set Open_vSwitch . other_config:hw-offload=true diff --git a/bindata/manifests/k8s-services/switchdev-configuration.service b/bindata/manifests/k8s-services/switchdev-configuration.service new file mode 100644 index 0000000000..f9a4f86dad --- /dev/null +++ b/bindata/manifests/k8s-services/switchdev-configuration.service @@ -0,0 +1,16 @@ +[Unit] +Description=Configures SRIOV NIC into switchdev mode +# Run service only when SR-IOV Network Operator is running +ConditionPathExists=/etc/sriov-netowrk-operator-run +# This service is used to move a SRIOV NIC into switchdev mode +Wants=network-pre.target +Before=network-pre.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/configure-switchdev.sh +StandardOutput=journal+console +StandardError=journal+console + +[Install] +WantedBy=network-online.target diff --git a/go.mod b/go.mod index d01a627608..08df963f31 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/genproto v0.0.0-20200610104632-a5b850bcf112 // indirect google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/yaml.v2 v2.3.0 k8s.io/api v0.19.0 k8s.io/apimachinery v0.19.0 k8s.io/client-go v0.19.0 diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 12facee248..f958c6f5d7 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -638,6 +638,8 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = registerPlugins(ns) if utils.ClusterType == utils.ClusterTypeOpenshift { pl = append(pl, McoPlugin) + } else { + pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) } diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index dd59de90fe..ffab8607d9 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -31,6 +31,7 @@ const ( GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" McoPlugin = "mco_plugin" + K8sPlugin = "k8s_plugin" ) // loadPlugin loads a single plugin from a file path diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go new file mode 100644 index 0000000000..7ebcf112b7 --- /dev/null +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -0,0 +1,157 @@ +package main + +import ( + "io/ioutil" + "path" + + "github.com/golang/glog" + "gopkg.in/yaml.v2" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type K8sPlugin struct { + PluginName string + SpecVersion string + services []serviceConf +} + +type serviceConf struct { + name string + serviceBin string +} + +const ( + k8sServicesManifestPath = "bindata/manifests/k8s-services" + ovsVSwitchdService = "ovs-vswitchd" + switchdevService = "switchdev-configuration" + configuresSwitchdevScript = "bindata/manifests/machine-config/files/configure-switchdev.sh.yaml" +) + +var ( + Plugin K8sPlugin + + serviceManager service.ServiceManager +) + +// Initialize our plugin and set up initial values +func init() { + Plugin = K8sPlugin{ + PluginName: "k8s_plugin", + SpecVersion: "1.0", + services: []serviceConf{{ + name: "switchdev-configuration", + serviceBin: "configure-switchdev.sh", + }}, + } + serviceManager = service.NewServiceManager("/host") +} + +// Name returns the name of the plugin +func (p *K8sPlugin) Name() string { + return p.PluginName +} + +// Spec returns the version of the spec expected by the plugin +func (p *K8sPlugin) Spec() string { + return p.SpecVersion +} + +// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateAdd()") + return p.OnNodeStateChange(nil, state) +} + +// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node +func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { + glog.Info("k8s-plugin OnNodeStateChange()") + needDrain = false + needReboot = false + + // Check services + var needServices bool + needServices, err = handleAddingServices() + if err != nil { + return + } + + if !needServices { + needServices, err = serviceManager.CheckServicesLoaded() + if err != nil { + return + } + } + needDrain = needServices + + // Check switchdev config + var update, remove bool + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + return + } + if remove { + needDrain = true + return + } + if update { + glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") + needDrain = true + needReboot = true + return + } + return +} + +// Apply config change +func (p *K8sPlugin) Apply() error { + glog.Info("k8s-plugin Apply()") + if err := serviceManager.LoadMissingServices(); err != nil { + return err + } + + return nil +} + +func handleAddingServices() (bool, error) { + needChange := false + + smServices := serviceManager.ServicesSet() + if _, ok := smServices[ovsVSwitchdService]; !ok { + ovsService, err := service.ServiceFromFile(ovsVSwitchdService, path.Join(k8sServicesManifestPath, ovsVSwitchdService+".service")) + if err != nil { + return false, err + } + needChange = true + _ = serviceManager.AddService(ovsService) + } + + if _, ok := smServices[switchdevService]; !ok { + switchdevService, err := service.ServiceFromFile(switchdevService, path.Join(k8sServicesManifestPath, switchdevService+".service")) + if err != nil { + return false, err + } + data, err := ioutil.ReadFile(configuresSwitchdevScript) + if err != nil { + return false, err + } + + var switchdevBinYaml struct { + Path string + Contents struct { + Inline string + } + } + if err := yaml.Unmarshal(data, &switchdevBinYaml); err != nil { + return false, err + } + switchdevService.Run = switchdevBinYaml.Path + switchdevService.RunBody = switchdevBinYaml.Contents.Inline + needChange = true + _ = serviceManager.AddService(switchdevService) + } + + return needChange, nil +} diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go index 215ae7fd80..74f3fb4c22 100644 --- a/pkg/plugins/mco/mco_plugin.go +++ b/pkg/plugins/mco/mco_plugin.go @@ -3,8 +3,6 @@ package main import ( "context" "encoding/json" - "fmt" - "io/ioutil" "os" "github.com/golang/glog" @@ -17,6 +15,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) type McoPlugin struct { @@ -28,7 +27,6 @@ type McoPlugin struct { const ( switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - switchDevConfPath = "/host/etc/switchdev.conf" nodeLabelPrefix = "node-role.kubernetes.io/" ) @@ -89,7 +87,7 @@ func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS } var update, remove bool - if update, remove, err = writeSwitchdevConfFile(new); err != nil { + if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) return } @@ -156,46 +154,3 @@ func (p *McoPlugin) Apply() error { glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) return nil } - -func writeSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { - _, err = os.Stat(switchDevConfPath) - if err != nil { - if os.IsNotExist(err) { - glog.V(2).Infof("writeSwitchdevConfFile(): file not existed, create it") - _, err = os.Create(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to create file: %v", err) - return - } - } else { - return - } - } - newContent := "" - for _, iface := range newState.Spec.Interfaces { - if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { - newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) - } - } - oldContent, err := ioutil.ReadFile(switchDevConfPath) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to read file: %v", err) - return - } - if newContent == string(oldContent) { - glog.V(2).Info("writeSwitchdevConfFile(): no update") - return - } - if newContent == "" { - remove = true - glog.V(2).Info("writeSwitchdevConfFile(): remove content in switchdev.conf") - } - update = true - glog.V(2).Infof("writeSwitchdevConfFile(): write %s to switchdev.conf", newContent) - err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666) - if err != nil { - glog.Errorf("writeSwitchdevConfFile(): fail to write file: %v", err) - return - } - return -} diff --git a/pkg/service/Service.go b/pkg/service/Service.go new file mode 100644 index 0000000000..2f4d79a03c --- /dev/null +++ b/pkg/service/Service.go @@ -0,0 +1,26 @@ +package service + +import ( + "io/ioutil" +) + +type Service struct { + Name string + Body string + Run string + RunBody string +} + +func ServiceFromFile(name, file string) (*Service, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + service := &Service{ + Name: name, + Body: string(data), + } + + return service, nil +} diff --git a/pkg/service/serviceManager.go b/pkg/service/serviceManager.go new file mode 100644 index 0000000000..e4280e6e19 --- /dev/null +++ b/pkg/service/serviceManager.go @@ -0,0 +1,119 @@ +package service + +import ( + "fmt" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "io/ioutil" + "os" + "os/exec" + "path" +) + +const ( + unitPath = "etc/systemd/system/" +) + +type ServiceManager interface { + AddService(*Service) error + LoadMissingServices() error + CheckServicesLoaded() (bool, error) + ServicesSet() map[string]bool +} + +type serviceManager struct { + chroot string + services map[string]*Service +} + +func NewServiceManager(chroot string) ServiceManager { + return &serviceManager{chroot, map[string]*Service{}} +} + +// AddService Add new service to system +func (sm *serviceManager) AddService(service *Service) error { + if _, ok := sm.services[service.Name]; ok { + return fmt.Errorf("serviceManager AddService(): service %s already exists", service.Name) + } + + sm.services[service.Name] = service + return nil +} + +func (sm *serviceManager) loadServiceOnHost(service *Service) error { + // Write service file + err := ioutil.WriteFile(path.Join(sm.chroot, unitPath, service.Name+".service"), []byte(service.Body), 0644) + if err != nil { + return err + } + + if service.Run != "" && service.RunBody != "" { + // Write service run + err = ioutil.WriteFile(path.Join(sm.chroot, service.Run), []byte(service.RunBody), 0755) + if err != nil { + return err + } + } + + // Change root dir + exit, err := utils.Chroot(sm.chroot) + if err != nil { + return fmt.Errorf("loadServiceOnHost(): service %q, failed with: %v", service.Name, err) + } + defer exit() + + // Enable service + cmd := exec.Command("systemctl", "enable", service.Name) + return cmd.Run() +} + +func (sm *serviceManager) isServiceLoaded(service string) (bool, error) { + _, err := os.Stat(path.Join(sm.chroot, unitPath, service+".service")) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +func (sm *serviceManager) CheckServicesLoaded() (bool, error) { + for service := range sm.services { + exists, err := sm.isServiceLoaded(service) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + } + + return true, nil +} + +func (sm *serviceManager) LoadMissingServices() error { + for serviceName, service := range sm.services { + exists, err := sm.isServiceLoaded(serviceName) + if err != nil { + return err + } + if exists { + continue + } + if err := sm.loadServiceOnHost(service); err != nil { + return err + } + } + + return nil +} + +func (sm *serviceManager) ServicesSet() map[string]bool { + services := map[string]bool{} + for service := range sm.services { + services[service] = true + } + + return services +} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go new file mode 100644 index 0000000000..818b53de2c --- /dev/null +++ b/pkg/utils/switchdev.go @@ -0,0 +1,58 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/golang/glog" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +const ( + switchDevConfPath = "/host/etc/switchdev.conf" +) + +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { + _, err = os.Stat(switchDevConfPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("WriteSwitchdevConfFile(): file not existed, create it") + _, err = os.Create(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to create file: %v", err) + return + } + } else { + return + } + } + newContent := "" + for _, iface := range newState.Spec.Interfaces { + if iface.EswitchMode == sriovnetworkv1.ESWITCHMODE_SWITCHDEV { + newContent = newContent + fmt.Sprintln(iface.PciAddress, iface.NumVfs) + } + } + oldContent, err := ioutil.ReadFile(switchDevConfPath) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to read file: %v", err) + return + } + if newContent == string(oldContent) { + glog.V(2).Info("WriteSwitchdevConfFile(): no update") + return + } + if newContent == "" { + remove = true + glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") + } + update = true + glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) + err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0666) + if err != nil { + glog.Errorf("WriteSwitchdevConfFile(): fail to write file: %v", err) + return + } + return +}