-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support changing Eswitch mode of SR-IOV NICs for kubernetes deployment
Signed-off-by: Mamduh Alassi <[email protected]>
- Loading branch information
Showing
11 changed files
with
627 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,269 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "io/ioutil" | ||
| "path" | ||
| "strings" | ||
|
|
||
| "github.com/golang/glog" | ||
| "gopkg.in/yaml.v2" | ||
|
|
||
| sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" | ||
| "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" | ||
| "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" | ||
| ) | ||
|
|
||
| type K8sPlugin struct { | ||
| PluginName string | ||
| SpecVersion string | ||
| } | ||
|
|
||
| type McoService struct { | ||
| Contents string | ||
| } | ||
|
|
||
| type McoFile struct { | ||
| Path string | ||
| Contents struct { | ||
| Inline string | ||
| } | ||
| } | ||
|
|
||
| type McoServiceInjection struct { | ||
| Dropins []struct { | ||
| Contents string | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| mcoManifestPath = "bindata/manifests/machine-config/" | ||
| switchdevUnits = mcoManifestPath + "switchdev-units/" | ||
| switchdevUnitFile = switchdevUnits + "switchdev-configuration.yaml" | ||
| networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" | ||
| configuresSwitchdevScript = mcoManifestPath + "files/configure-switchdev.sh.yaml" | ||
| ovsVSwitchdUnitFile = mcoManifestPath + "ovs-units/ovs-vswitchd.service.yaml" | ||
|
|
||
| switchdevService = "switchdev-configuration" | ||
| networkManagerService = "NetworkManager" | ||
|
|
||
| switchdevServicePath = "/etc/systemd/system/" + switchdevService + ".service" | ||
| systemdDir = "/usr/lib/systemd/system" | ||
| ) | ||
|
|
||
| var ( | ||
| Plugin K8sPlugin | ||
|
|
||
| serviceManager service.ServiceManager | ||
|
|
||
| ovsServiceName = "openvswitch" | ||
|
|
||
| needServices bool | ||
| needInjectServices bool | ||
| ) | ||
|
|
||
| // Initialize our plugin and set up initial values | ||
| func init() { | ||
| Plugin = K8sPlugin{ | ||
| PluginName: "k8s_plugin", | ||
| SpecVersion: "1.0", | ||
| } | ||
|
|
||
| // ovs service name is different on ubuntu | ||
| serviceManager = service.NewServiceManager("/host") | ||
| data, err := ioutil.ReadFile("/host/etc/os-release") | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| osInfo := string(data) | ||
| if strings.Contains(osInfo, "ubuntu") { | ||
| ovsServiceName = "openvswitch-switch" | ||
| } | ||
| } | ||
|
|
||
| // Name returns the name of the plugin | ||
| func (p *K8sPlugin) Name() string { | ||
| return p.PluginName | ||
| } | ||
|
|
||
| // Spec returns the version of the spec expected by the plugin | ||
| func (p *K8sPlugin) Spec() string { | ||
| return p.SpecVersion | ||
| } | ||
|
|
||
| // OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node | ||
| func (p *K8sPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { | ||
| glog.Info("k8s-plugin OnNodeStateAdd()") | ||
| return p.OnNodeStateChange(nil, state) | ||
| } | ||
|
|
||
| // OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node | ||
| func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { | ||
| glog.Info("k8s-plugin OnNodeStateChange()") | ||
| needDrain = false | ||
| needReboot = false | ||
|
|
||
| // Check services | ||
| needServices, err = handleServices() | ||
| if err != nil { | ||
| return | ||
| } | ||
|
|
||
| if !needServices { | ||
| needServices, err = serviceManager.CheckServicesLoaded() | ||
| if err != nil { | ||
| return | ||
| } | ||
| } | ||
| needDrain = needServices | ||
|
|
||
| needInjectServices, err = handleServiceInjections() | ||
| if err != nil { | ||
| return false, false, err | ||
| } | ||
| if !needInjectServices { | ||
| needInjectServices, err = serviceManager.CheckServiceInjections() | ||
| } | ||
| needDrain = needDrain || needInjectServices | ||
|
|
||
| // Check switchdev config | ||
| var update, remove bool | ||
| if new == nil { | ||
| return | ||
| } | ||
| if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { | ||
| glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) | ||
| return | ||
| } | ||
| if remove { | ||
| glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") | ||
| needDrain = true | ||
| needReboot = true | ||
| return | ||
| } | ||
| if update { | ||
| glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") | ||
| needDrain = true | ||
| needReboot = true | ||
| return | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // Apply config change | ||
| func (p *K8sPlugin) Apply() error { | ||
| glog.Info("k8s-plugin Apply()") | ||
| if needServices { | ||
| if err := serviceManager.LoadMissingServices(); err != nil { | ||
| return err | ||
| } | ||
| needServices = false | ||
| } | ||
|
|
||
| if needInjectServices { | ||
| if err := serviceManager.InjectServices(); err != nil { | ||
| return err | ||
| } | ||
| needInjectServices = false | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func handleServices() (bool, error) { | ||
| needChange := false | ||
| smServices := serviceManager.ServicesSet() | ||
| // Check switchdev service | ||
| if _, ok := smServices[switchdevService]; !ok { | ||
| needChange = true | ||
| data, err := ioutil.ReadFile(switchdevUnitFile) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| var switchdevServiceContent McoService | ||
| if err := yaml.Unmarshal(data, &switchdevServiceContent); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| data, err = ioutil.ReadFile(configuresSwitchdevScript) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| var switchdevRunFile McoFile | ||
| if err := yaml.Unmarshal(data, &switchdevRunFile); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| switchdevService := &service.Service{ | ||
| Name: switchdevService, | ||
| Path: switchdevServicePath, | ||
| Body: switchdevServiceContent.Contents, | ||
| Run: switchdevRunFile.Path, | ||
| RunBody: switchdevRunFile.Contents.Inline, | ||
| } | ||
|
|
||
| if err = serviceManager.AddService(switchdevService); err != nil { | ||
| return false, err | ||
| } | ||
| } | ||
|
|
||
| return needChange, nil | ||
| } | ||
|
|
||
| func handleServiceInjections() (bool, error) { | ||
| needChange := false | ||
| smServices := serviceManager.InjectableServicesSet() | ||
|
|
||
| // Check ovs service exists | ||
| if _, ok := smServices[ovsServiceName]; !ok { | ||
| needChange = true | ||
|
|
||
| data, err := ioutil.ReadFile(ovsVSwitchdUnitFile) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| var ovsServiceContent McoServiceInjection | ||
| if err := yaml.Unmarshal(data, &ovsServiceContent); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| ovsService, err := service.NewInjectableService(ovsServiceName, | ||
| path.Join(systemdDir, ovsServiceName+".service"), | ||
| ovsServiceContent.Dropins[0].Contents) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| if err = serviceManager.AddInjectableService(ovsService); err != nil { | ||
| return false, err | ||
| } | ||
| } | ||
|
|
||
| // Check NetworkManager service exists | ||
| if _, ok := smServices[networkManagerService]; !ok { | ||
| needChange = true | ||
|
|
||
| data, err := ioutil.ReadFile(networkManagerUnitFile) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| var networkManagerServiceContent McoServiceInjection | ||
| if err := yaml.Unmarshal(data, &networkManagerServiceContent); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| networkMgrService, err := service.NewInjectableService(networkManagerService, | ||
| path.Join(systemdDir, networkManagerService+".service"), | ||
| networkManagerServiceContent.Dropins[0].Contents) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| if err = serviceManager.AddInjectableService(networkMgrService); err != nil { | ||
| return false, err | ||
| } | ||
| } | ||
|
|
||
| return needChange, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.