From 34299d6f1d1f838b0cf885493136785e9770076a Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Thu, 6 May 2021 13:36:55 +0800 Subject: [PATCH 1/2] Initialize sriovnetworkpoolconfig API --- api/v1/sriovnetworkpoolconfig_types.go | 49 ++++++++ api/v1/zz_generated.deepcopy.go | 105 ++++++++++++++++++ ....openshift.io_sriovnetworkpoolconfigs.yaml | 66 +++++++++++ config/crd/kustomization.yaml | 9 +- ...ainjection_in_sriovnetworkpoolconfigs.yaml | 8 ++ .../webhook_in_sriovnetworkpoolconfigs.yaml | 17 +++ .../sriovnetworkpoolconfig_editor_role.yaml | 24 ++++ .../sriovnetworkpoolconfig_viewer_role.yaml | 20 ++++ ...riovnetwork_v1_sriovnetworkpoolconfig.yaml | 7 ++ .../sriovnetworknodeconfig_controller.go | 47 ++++++++ main.go | 12 +- 11 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 api/v1/sriovnetworkpoolconfig_types.go create mode 100644 config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml create mode 100644 config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml create mode 100644 config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml create mode 100644 config/rbac/sriovnetworkpoolconfig_editor_role.yaml create mode 100644 config/rbac/sriovnetworkpoolconfig_viewer_role.yaml create mode 100644 config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml create mode 100644 controllers/sriovnetworknodeconfig_controller.go diff --git a/api/v1/sriovnetworkpoolconfig_types.go b/api/v1/sriovnetworkpoolconfig_types.go new file mode 100644 index 000000000..c79473333 --- /dev/null +++ b/api/v1/sriovnetworkpoolconfig_types.go @@ -0,0 +1,49 @@ +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SriovNetworkPoolConfigSpec defines the desired state of SriovNetworkPoolConfig +type SriovNetworkPoolConfigSpec struct { + // OvsHardwareOffloadConfig describes the OVS HWOL configuration for selected Nodes + OvsHardwareOffloadConfig OvsHardwareOffloadConfig `json:"ovsHardwareOffloadConfig,omitempty"` +} + +type OvsHardwareOffloadConfig struct { + // Name is mandatory and must be unique. + // On Kubernetes: + // Name is the name of OvsHardwareOffloadConfig + // On OpenShift: + // Name is the name of MachineConfigPool to be enabled with OVS hardware offload + Name string `json:"name,omitempty"` +} + +// SriovNetworkPoolConfigStatus defines the observed state of SriovNetworkPoolConfig +type SriovNetworkPoolConfigStatus struct { +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// SriovNetworkPoolConfig is the Schema for the sriovnetworkpoolconfigs API +type SriovNetworkPoolConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SriovNetworkPoolConfigSpec `json:"spec,omitempty"` + Status SriovNetworkPoolConfigStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SriovNetworkPoolConfigList contains a list of SriovNetworkPoolConfig +type SriovNetworkPoolConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SriovNetworkPoolConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SriovNetworkPoolConfig{}, &SriovNetworkPoolConfigList{}) +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0cac719ca..99b80021a 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -127,6 +127,21 @@ func (in Interfaces) DeepCopy() Interfaces { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OvsHardwareOffloadConfig) DeepCopyInto(out *OvsHardwareOffloadConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OvsHardwareOffloadConfig. +func (in *OvsHardwareOffloadConfig) DeepCopy() *OvsHardwareOffloadConfig { + if in == nil { + return nil + } + out := new(OvsHardwareOffloadConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SriovIBNetwork) DeepCopyInto(out *SriovIBNetwork) { *out = *in @@ -500,6 +515,96 @@ func (in *SriovNetworkNodeStateStatus) DeepCopy() *SriovNetworkNodeStateStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SriovNetworkPoolConfig) DeepCopyInto(out *SriovNetworkPoolConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkPoolConfig. +func (in *SriovNetworkPoolConfig) DeepCopy() *SriovNetworkPoolConfig { + if in == nil { + return nil + } + out := new(SriovNetworkPoolConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SriovNetworkPoolConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SriovNetworkPoolConfigList) DeepCopyInto(out *SriovNetworkPoolConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SriovNetworkPoolConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkPoolConfigList. +func (in *SriovNetworkPoolConfigList) DeepCopy() *SriovNetworkPoolConfigList { + if in == nil { + return nil + } + out := new(SriovNetworkPoolConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SriovNetworkPoolConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SriovNetworkPoolConfigSpec) DeepCopyInto(out *SriovNetworkPoolConfigSpec) { + *out = *in + out.OvsHardwareOffloadConfig = in.OvsHardwareOffloadConfig +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkPoolConfigSpec. +func (in *SriovNetworkPoolConfigSpec) DeepCopy() *SriovNetworkPoolConfigSpec { + if in == nil { + return nil + } + out := new(SriovNetworkPoolConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SriovNetworkPoolConfigStatus) DeepCopyInto(out *SriovNetworkPoolConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkPoolConfigStatus. +func (in *SriovNetworkPoolConfigStatus) DeepCopy() *SriovNetworkPoolConfigStatus { + if in == nil { + return nil + } + out := new(SriovNetworkPoolConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SriovNetworkSpec) DeepCopyInto(out *SriovNetworkSpec) { *out = *in diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml new file mode 100644 index 000000000..eba8c8804 --- /dev/null +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -0,0 +1,66 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkPoolConfig + listKind: SriovNetworkPoolConfigList + plural: sriovnetworkpoolconfigs + singular: sriovnetworkpoolconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkPoolConfig is the Schema for the sriovnetworkpoolconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SriovNetworkPoolConfigSpec defines the desired state of SriovNetworkPoolConfig + properties: + ovsHardwareOffloadConfig: + description: OvsHardwareOffloadConfig describes the OVS HWOL configuration + for selected Nodes + properties: + name: + description: 'Name is mandatory and must be unique. On Kubernetes: + Name is the name of OvsHardwareOffloadConfig On OpenShift: Name + is the name of MachineConfigPool to be enabled with OVS hardware + offload' + type: string + type: object + type: object + status: + description: SriovNetworkPoolConfigStatus defines the observed state of + SriovNetworkPoolConfig + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f0be96873..6e4c3d59a 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,7 +7,8 @@ resources: - bases/sriovnetwork.openshift.io_sriovibnetworks.yaml - bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml - bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml -#+kubebuilder:scaffold:crdkustomizeresource +- bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +# +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. @@ -17,7 +18,8 @@ patchesStrategicMerge: #- patches/webhook_in_sriovibnetworks.yaml #- patches/webhook_in_sriovnetworknodepolicies.yaml #- patches/webhook_in_sriovoperatorconfigs.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch +#- patches/webhook_in_sriovnetworkpoolconfigs.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD @@ -26,7 +28,8 @@ patchesStrategicMerge: #- patches/cainjection_in_sriovibnetworks.yaml #- patches/cainjection_in_sriovnetworknodepolicies.yaml #- patches/cainjection_in_sriovoperatorconfigs.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch +#- patches/cainjection_in_sriovnetworkpoolconfigs.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: diff --git a/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml b/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml new file mode 100644 index 000000000..8410ee441 --- /dev/null +++ b/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io diff --git a/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml b/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml new file mode 100644 index 000000000..4dd610c65 --- /dev/null +++ b/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/rbac/sriovnetworkpoolconfig_editor_role.yaml b/config/rbac/sriovnetworkpoolconfig_editor_role.yaml new file mode 100644 index 000000000..342506ecf --- /dev/null +++ b/config/rbac/sriovnetworkpoolconfig_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit sriovnetworkpoolconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sriovnetworkpoolconfig-editor-role +rules: +- apiGroups: + - sriovnetwork.openshift.io + resources: + - sriovnetworkpoolconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - sriovnetwork.openshift.io + resources: + - sriovnetworkpoolconfigs/status + verbs: + - get diff --git a/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml b/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml new file mode 100644 index 000000000..d669d324e --- /dev/null +++ b/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view sriovnetworkpoolconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sriovnetworkpoolconfig-viewer-role +rules: +- apiGroups: + - sriovnetwork.openshift.io + resources: + - sriovnetworkpoolconfigs + verbs: + - get + - list + - watch +- apiGroups: + - sriovnetwork.openshift.io + resources: + - sriovnetworkpoolconfigs/status + verbs: + - get diff --git a/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml b/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml new file mode 100644 index 000000000..c02325e9a --- /dev/null +++ b/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml @@ -0,0 +1,7 @@ +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkPoolConfig +metadata: + name: sriovnetworkpoolconfig-sample +spec: + # Add fields here + foo: bar diff --git a/controllers/sriovnetworknodeconfig_controller.go b/controllers/sriovnetworknodeconfig_controller.go new file mode 100644 index 000000000..b51665e3f --- /dev/null +++ b/controllers/sriovnetworknodeconfig_controller.go @@ -0,0 +1,47 @@ +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +// SriovNetworkNodeConfigReconciler reconciles a SriovNetworkNodeConfig object +type SriovNetworkNodeConfigReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs/status,verbs=get;update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the SriovNetworkNodeConfig object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile +func (r *SriovNetworkNodeConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + _ = context.Background() + _ = r.Log.WithValues("sriovnetworknodeconfig", req.NamespacedName) + + // your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SriovNetworkNodeConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sriovnetworkv1.SriovNetworkNodeConfig{}). + Complete(r) +} diff --git a/main.go b/main.go index cdb4fbf66..184fe37e5 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "k8s.io/apimachinery/pkg/api/errors" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -126,7 +127,16 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "SriovOperatorConfig") os.Exit(1) } - //+kubebuilder:scaffold:builder + if err = (&controllers.SriovNetworkPoolConfigReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("SriovNetworkPoolConfig"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SriovNetworkPoolConfig") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + // Create a default SriovNetworkNodePolicy err = createDefaultPolicy(ctrl.GetConfigOrDie()) if err != nil { From 463afa98b264b22f5ea96c0054643155556e493e Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Thu, 6 May 2021 16:39:43 +0800 Subject: [PATCH 2/2] OpenShift: enable OVS HW offload for multiple MCPs One node can only be added in one custom MCP pool by design of MCO. Since current ovs-hw-offload MCs are applied to hardcoded MCP ovs-hw-offload-worker, other existing custome MCPs cannot be enabled with ovs hw offload configuration. This commits allows ovs-hw-offload MCs be applied to multiple MCPs as specified in default SriovNetworkNodeConfig. --- Makefile | 2 +- api/v1/helper.go | 9 +- controllers/helper.go | 30 +-- controllers/sriovibnetwork_controller.go | 8 +- controllers/sriovnetwork_controller.go | 8 +- .../sriovnetworknodeconfig_controller.go | 47 ----- .../sriovnetworkpoolconfig_controller.go | 175 ++++++++++++++++++ .../sriovnetworkpoolconfig_controller_test.go | 63 +++++++ controllers/sriovoperatorconfig_controller.go | 108 ----------- .../sriovoperatorconfig_controller_test.go | 61 ------ controllers/suite_test.go | 12 ++ main.go | 1 - pkg/daemon/daemon.go | 21 +-- pkg/daemon/plugin.go | 1 - pkg/plugins/generic/generic_plugin.go | 57 +++--- pkg/plugins/k8s/k8s_plugin.go | 32 +--- pkg/plugins/mco/mco_plugin.go | 150 --------------- pkg/utils/switchdev.go | 6 +- 18 files changed, 325 insertions(+), 466 deletions(-) delete mode 100644 controllers/sriovnetworknodeconfig_controller.go create mode 100644 controllers/sriovnetworkpoolconfig_controller.go create mode 100644 controllers/sriovnetworkpoolconfig_controller_test.go delete mode 100644 pkg/plugins/mco/mco_plugin.go diff --git a/Makefile b/Makefile index 799de204c..17986ad2e 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/api/v1/helper.go b/api/v1/helper.go index 2042465af..4559f0451 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -24,10 +24,11 @@ import ( ) const ( - LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" - FINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" - ESWITCHMODE_LEGACY = "legacy" - ESWITCHMODE_SWITCHDEV = "switchdev" + LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" + NETATTDEFFINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" + POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io" + ESWITCHMODE_LEGACY = "legacy" + ESWITCHMODE_SWITCHDEV = "switchdev" ) const invalidVfIndex = -1 diff --git a/controllers/helper.go b/controllers/helper.go index 0a3068351..947125c1e 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -22,21 +22,21 @@ import ( ) const ( - ResyncPeriod = 5 * time.Minute - DEFAULT_CONFIG_NAME = "default" - CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" - INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" - OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" - SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" - INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" - OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" - DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" - PLUGIN_PATH = "./bindata/manifests/plugins" - DAEMON_PATH = "./bindata/manifests/daemon" - DEFAULT_POLICY_NAME = "default" - CONFIGMAP_NAME = "device-plugin-config" - DP_CONFIG_FILENAME = "config.json" - HwOffloadNodeLabel = "ovs-hw-offload-worker" + ResyncPeriod = 5 * time.Minute + DEFAULT_CONFIG_NAME = "default" + CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" + INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" + OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" + SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" + INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" + OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" + DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" + PLUGIN_PATH = "./bindata/manifests/plugins" + DAEMON_PATH = "./bindata/manifests/daemon" + DEFAULT_POLICY_NAME = "default" + CONFIGMAP_NAME = "device-plugin-config" + DP_CONFIG_FILENAME = "config.json" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX = "ovs-hw-offload" linkTypeEthernet = "ether" linkTypeInfiniband = "infiniband" diff --git a/controllers/sriovibnetwork_controller.go b/controllers/sriovibnetwork_controller.go index 6741d5806..5313efb93 100644 --- a/controllers/sriovibnetwork_controller.go +++ b/controllers/sriovibnetwork_controller.go @@ -81,15 +81,15 @@ func (r *SriovIBNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Reque // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { - instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.FINALIZERNAME) + if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } } } else { // The object is being deleted - if sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { + if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { // our finalizer is present, so lets handle any external dependency reqLogger.Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.Spec.NetworkNamespace, "Name", instance.Name) if err := instance.DeleteNetAttDef(r.Client); err != nil { @@ -98,7 +98,7 @@ func (r *SriovIBNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Reque return reconcile.Result{}, err } // remove our finalizer from the list and update it. - instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } diff --git a/controllers/sriovnetwork_controller.go b/controllers/sriovnetwork_controller.go index 5e6930f01..6795ae35f 100644 --- a/controllers/sriovnetwork_controller.go +++ b/controllers/sriovnetwork_controller.go @@ -82,15 +82,15 @@ func (r *SriovNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { - instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.FINALIZERNAME) + if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } } } else { // The object is being deleted - if sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { + if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { // our finalizer is present, so lets handle any external dependency reqLogger.Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.Spec.NetworkNamespace, "Name", instance.Name) if err := instance.DeleteNetAttDef(r.Client); err != nil { @@ -99,7 +99,7 @@ func (r *SriovNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, err } // remove our finalizer from the list and update it. - instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } diff --git a/controllers/sriovnetworknodeconfig_controller.go b/controllers/sriovnetworknodeconfig_controller.go deleted file mode 100644 index b51665e3f..000000000 --- a/controllers/sriovnetworknodeconfig_controller.go +++ /dev/null @@ -1,47 +0,0 @@ -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovNetworkNodeConfigReconciler reconciles a SriovNetworkNodeConfig object -type SriovNetworkNodeConfigReconciler struct { - client.Client - Log logr.Logger - Scheme *runtime.Scheme -} - -//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs/status,verbs=get;update;patch - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the SriovNetworkNodeConfig object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile -func (r *SriovNetworkNodeConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - _ = context.Background() - _ = r.Log.WithValues("sriovnetworknodeconfig", req.NamespacedName) - - // your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *SriovNetworkNodeConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&sriovnetworkv1.SriovNetworkNodeConfig{}). - Complete(r) -} diff --git a/controllers/sriovnetworkpoolconfig_controller.go b/controllers/sriovnetworkpoolconfig_controller.go new file mode 100644 index 000000000..b3ff6c4ad --- /dev/null +++ b/controllers/sriovnetworkpoolconfig_controller.go @@ -0,0 +1,175 @@ +package controllers + +import ( + "bytes" + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +// SriovNetworkPoolConfigReconciler reconciles a SriovNetworkPoolConfig object +type SriovNetworkPoolConfigReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the SriovNetworkPoolConfig object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile +func (r *SriovNetworkPoolConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithValues("sriovnetworkpoolconfig", req.NamespacedName) + logger.Info("Reconciling") + + // // Fetch SriovNetworkPoolConfig + instance := &sriovnetworkv1.SriovNetworkPoolConfig{} + err := r.Get(context.TODO(), req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + logger.Info("instance not found") + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // examine DeletionTimestamp to determine if object is under deletion + if instance.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !sriovnetworkv1.StringInArray(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.POOLCONFIGFINALIZERNAME) + if err := r.Update(context.Background(), instance); err != nil { + return reconcile.Result{}, err + } + } + if utils.ClusterType == utils.ClusterTypeOpenshift { + if err = r.syncOvsHardwareOffloadMachineConfigs(instance, false); err != nil { + return reconcile.Result{}, err + } + } + } else { + // The object is being deleted + if sriovnetworkv1.StringInArray(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) { + // our finalizer is present, so lets handle any external dependency + logger.Info("delete SriovNetworkPoolConfig CR", "Namespace", instance.Namespace, "Name", instance.Name) + if utils.ClusterType == utils.ClusterTypeOpenshift { + if err = r.syncOvsHardwareOffloadMachineConfigs(instance, true); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return reconcile.Result{}, err + } + } + // remove our finalizer from the list and update it. + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) + if err := r.Update(context.Background(), instance); err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, err + } + + return reconcile.Result{RequeueAfter: ResyncPeriod}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SriovNetworkPoolConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sriovnetworkv1.SriovNetworkPoolConfig{}). + Complete(r) +} + +func (r *SriovNetworkPoolConfigReconciler) syncOvsHardwareOffloadMachineConfigs(nc *sriovnetworkv1.SriovNetworkPoolConfig, deletion bool) error { + logger := log.Log.WithName("syncOvsHardwareOffloadMachineConfigs") + + mcpName := nc.Spec.OvsHardwareOffloadConfig.Name + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + + foundMC := &mcfgv1.MachineConfig{} + mcp := &mcfgv1.MachineConfigPool{} + + if mcpName == "" { + logger.Info("MachineConfigPool name is not defined in SriovNetworkPoolConfig", nc.GetNamespace(), nc.GetName()) + return nil + } + + if mcpName == "master" { + logger.Info("Master nodes are selected by OvsHardwareOffloadConfig.Name, ignoring.") + return nil + } + + err := r.Get(context.TODO(), types.NamespacedName{Name: mcpName}, mcp) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("MachineConfigPool %s doesn't exist: %v", mcpName, err) + } + } + + data := render.MakeRenderData() + mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, mcpName, true, &data) + if err != nil { + return err + } + + err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) + if err != nil { + if errors.IsNotFound(err) { + if deletion { + logger.Info("MachineConfig has already been deleted") + } else { + err = r.Create(context.TODO(), mc) + if err != nil { + return fmt.Errorf("Couldn't create MachineConfig: %v", err) + } + logger.Info("Created MachineConfig CR in MachineConfigPool", mcName, mcpName) + } + } else { + return fmt.Errorf("Failed to get MachineConfig: %v", err) + } + } else { + if deletion { + logger.Info("offload disabled, delete MachineConfig") + err = r.Delete(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't delete MachineConfig: %v", err) + } + } else { + if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { + logger.Info("MachineConfig already exists, updating") + err = r.Update(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't update MachineConfig: %v", err) + } + } else { + logger.Info("No content change, skip updating MC") + } + } + } + + return nil +} diff --git a/controllers/sriovnetworkpoolconfig_controller_test.go b/controllers/sriovnetworkpoolconfig_controller_test.go new file mode 100644 index 000000000..e7d38e84a --- /dev/null +++ b/controllers/sriovnetworkpoolconfig_controller_test.go @@ -0,0 +1,63 @@ +package controllers + +import ( + goctx "context" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +var _ = Describe("Operator", func() { + Context("When is up", func() { + It("should be able to create machine config for MachineConfigPool specified in sriov pool config", func() { + config := &sriovnetworkv1.SriovNetworkPoolConfig{} + config.SetNamespace(testNamespace) + config.SetName("ovs-hw-offload-config") + mcpName := "worker" + mc := &mcfgv1.MachineConfig{} + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp := &mcfgv1.MachineConfigPool{} + err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp = &mcfgv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcpName, + }, + Spec: mcfgv1.MachineConfigPoolSpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/worker": "", + }, + }, + }, + } + err = k8sClient.Create(goctx.TODO(), mcp) + Expect(err).NotTo(HaveOccurred()) + + config.Spec.OvsHardwareOffloadConfig = sriovnetworkv1.OvsHardwareOffloadConfig{ + Name: mcpName, + } + err = k8sClient.Create(goctx.TODO(), config) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() error { + mc := &mcfgv1.MachineConfig{} + err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + if err != nil { + return err + } + return nil + }, timeout*3, interval).Should(Succeed()) + }) + }) +}) diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 60614966e..fa8018637 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -17,13 +17,10 @@ limitations under the License. package controllers import ( - "bytes" "context" "fmt" "os" - "github.com/openshift/machine-config-operator/lib/resourcemerge" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -119,12 +116,6 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. return reconcile.Result{}, err } - if utils.ClusterType == utils.ClusterTypeOpenshift { - if err = r.syncOffloadMachineConfig(defaultConfig); err != nil { - return reconcile.Result{}, err - } - } - return reconcile.Result{RequeueAfter: ResyncPeriod}, nil } @@ -306,102 +297,3 @@ func (r *SriovOperatorConfigReconciler) syncK8sResource(cr *sriovnetworkv1.Sriov } return nil } - -func (r *SriovOperatorConfigReconciler) syncOffloadMachineConfig(dc *sriovnetworkv1.SriovOperatorConfig) error { - logger := log.Log.WithName("syncOffloadMachineConfig") - var err error - - logger.Info("Start to render MachineConfig and MachineConfigPool for OVS HW offloading") - data := render.MakeRenderData() - data.Data["HwOffloadNodeLabel"] = HwOffloadNodeLabel - mcName := "00-" + HwOffloadNodeLabel - mcpName := HwOffloadNodeLabel - mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, HwOffloadNodeLabel, dc.Spec.EnableOvsOffload, &data) - if err != nil { - return err - } - mcpRaw, err := render.RenderTemplate("bindata/manifests/switchdev-config/machineconfigpool.yaml", &data) - if err != nil { - return err - } - mcp := &mcfgv1.MachineConfigPool{} - if len(mcpRaw) != 1 { - return fmt.Errorf("Invalid MachineConfigPool CR template") - } - err = r.Scheme.Convert(mcpRaw[0], mcp, context.TODO()) - if err != nil { - return err - } - - foundMC := &mcfgv1.MachineConfig{} - foundMCP := &mcfgv1.MachineConfigPool{} - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mc) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfig: %v", err) - } - logger.Info("Created MachineConfig CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfig: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MC") - } - } else { - logger.Info("offload disabled, delete MachineConfig") - err = r.Delete(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfig: %v", err) - } - } - } - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcpName}, foundMCP) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mcp) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfigPool: %v", err) - } - logger.Info("Created MachineConfigPool CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfigPool: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - modified := resourcemerge.BoolPtr(false) - resourcemerge.EnsureMachineConfigPool(modified, foundMCP, *mcp) - if *modified { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MCP") - } - } else { - logger.Info("offload disabled, delete MachineConfigPool") - err = r.Delete(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfigPool: %v", err) - } - } - } - return nil -} diff --git a/controllers/sriovoperatorconfig_controller_test.go b/controllers/sriovoperatorconfig_controller_test.go index b1b0a33a8..bd2919e15 100644 --- a/controllers/sriovoperatorconfig_controller_test.go +++ b/controllers/sriovoperatorconfig_controller_test.go @@ -5,13 +5,11 @@ import ( admv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" util "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" @@ -174,64 +172,5 @@ var _ = Describe("Operator", func() { }, timeout*10, interval).Should(Equal(config.Spec.ConfigDaemonNodeSelector)) }) - It("should be able to create/remove hw offload MCO and MC", func() { - config := &sriovnetworkv1.SriovOperatorConfig{} - err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", interval, timeout) - Expect(err).NotTo(HaveOccurred()) - By("by default") - mcName := "00-" + HwOffloadNodeLabel - mc := &mcfgv1.MachineConfig{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - mcpName := HwOffloadNodeLabel - mcp := &mcfgv1.MachineConfigPool{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - By("set EnableOvsOffload to true to create MC and MCP resources") - config.Spec.EnableOvsOffload = true - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - Eventually(func() error { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - - By("set EnableOvsOffload to false to delete MC and MCP resources") - config.Spec.EnableOvsOffload = false - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() bool { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - Eventually(func() bool { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - }) }) }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index f7ff9d4e7..1ccbaf36c 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -114,6 +114,12 @@ var _ = BeforeSuite(func(done Done) { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&SriovNetworkPoolConfigReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + os.Setenv("RESOURCE_PREFIX", "openshift.io") os.Setenv("NAMESPACE", "openshift-sriov-network-operator") os.Setenv("ENABLE_ADMISSION_CONTROLLER", "true") @@ -153,6 +159,12 @@ var _ = BeforeSuite(func(done Done) { LogLevel: 2, } Expect(k8sClient.Create(context.TODO(), config)).Should(Succeed()) + + poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{} + poolConfig.SetNamespace(testNamespace) + poolConfig.SetName(DEFAULT_CONFIG_NAME) + poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{} + Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed()) close(done) }, 60) diff --git a/main.go b/main.go index 184fe37e5..9c44da3bb 100644 --- a/main.go +++ b/main.go @@ -129,7 +129,6 @@ func main() { } if err = (&controllers.SriovNetworkPoolConfigReconciler{ Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("SriovNetworkPoolConfig"), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SriovNetworkPoolConfig") diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 4288989e8..dec1e4d56 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -464,7 +464,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { reqReboot := false reqDrain := false - mcoReboot := false for k, p := range dn.LoadedPlugins { d, r := false, false if dn.nodeState.GetName() == "" { @@ -476,11 +475,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.Errorf("nodeStateSyncHandler(): plugin %s error: %v", k, err) return err } - - if p.Name() == "mco_plugin" && d && !r { - mcoReboot = true - } - glog.V(0).Infof("nodeStateSyncHandler(): plugin %s: reqDrain %v, reqReboot %v", k, d, r) reqDrain = reqDrain || d reqReboot = reqReboot || r @@ -488,7 +482,7 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.V(0).Infof("nodeStateSyncHandler(): reqDrain %v, reqReboot %v disableDrain %v", reqDrain, reqReboot, dn.disableDrain) for k, p := range dn.LoadedPlugins { - if k != GenericPlugin && k != McoPlugin { + if k != GenericPlugin { err := p.Apply() if err != nil { glog.Errorf("nodeStateSyncHandler(): plugin %s fail to apply: %v", k, err) @@ -496,15 +490,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } } } - if mcoReboot { - glog.V(0).Infof("nodeStateSyncHandler(): MCO will trigger reboot") - err = dn.LoadedPlugins[McoPlugin].Apply() - if err != nil { - glog.Errorf("nodeStateSyncHandler(): mco_plugin fail to apply: %v", err) - return err - } - return nil - } if utils.ClusterType == utils.ClusterTypeOpenshift { if err = dn.getNodeMachinePool(); err != nil { return err @@ -649,9 +634,7 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = append(pl, VirtualPlugin) } else { pl = registerPlugins(ns) - if utils.ClusterType == utils.ClusterTypeOpenshift { - pl = append(pl, McoPlugin) - } else { + if utils.ClusterType != utils.ClusterTypeOpenshift { pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index ffab8607d..eb4458f1b 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -30,7 +30,6 @@ const ( SpecVersion = "1.0" GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" - McoPlugin = "mco_plugin" K8sPlugin = "k8s_plugin" ) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 966fdc8e0..97d3d07ac 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -59,18 +59,10 @@ func (p *GenericPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeSta p.DesireState = state needDrain = needDrainNode(state.Spec.Interfaces, state.Status.Interfaces) + needReboot = needRebootNode(state, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(state) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -84,18 +76,10 @@ func (p *GenericPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkN p.DesireState = new needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces) + needReboot = needRebootNode(new, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(new) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -212,3 +196,32 @@ func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.Int } return } + +func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint) (needReboot bool) { + needReboot = false + if *loadVfioDriver != loaded { + if needVfioDriver(state) { + *loadVfioDriver = loading + update, err := tryEnableIommuInKernelArgs() + if err != nil { + glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err) + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args") + } + needReboot = needReboot || update + } + } + + if utils.IsSwitchdevModeSpec(state.Spec) { + update, err := utils.WriteSwitchdevConfFile(state) + if err != nil { + glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file") + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration") + } + needReboot = needReboot || update + } + return +} diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index c26d82b42..e3def1e00 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -117,34 +117,18 @@ func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS p.updateTarget.reset() // TODO add check for enableOvsOffload in OperatorConfig later - // Update services if switchdev not required - if utils.IsSwitchdevModeSpec(new.Spec) { - // Check services - err = p.servicesStateUpdate() - if err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) - return - } - } - - // Check switchdev config - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + // Update services if switchdev required + if !utils.IsSwitchdevModeSpec(new.Spec) { return } - if remove { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") - needDrain = true - needReboot = true - return - } - if update { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true + + // Check services + err = p.servicesStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) return } + if p.updateTarget.needUpdate() { needDrain = true if p.updateTarget.switchdevUdevScript { diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go deleted file mode 100644 index b63c8dfe3..000000000 --- a/pkg/plugins/mco/mco_plugin.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - - "github.com/golang/glog" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -type McoPlugin struct { - PluginName string - SpecVersion string - DesireState *sriovnetworkv1.SriovNetworkNodeState - LastState *sriovnetworkv1.SriovNetworkNodeState -} - -const ( - switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - nodeLabelPrefix = "node-role.kubernetes.io/" -) - -var nodeName string -var Plugin McoPlugin -var kubeclient *kubernetes.Clientset -var switchdevConfigured bool - -// Initialize our plugin and set up initial values -func init() { - Plugin = McoPlugin{ - PluginName: "mco_plugin", - SpecVersion: "1.0", - } - - var config *rest.Config - var err error - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig != "" { - config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - } else { - // creates the in-cluster config - config, err = rest.InClusterConfig() - } - - if err != nil { - panic(err.Error()) - } - kubeclient = kubernetes.NewForConfigOrDie(config) -} - -// Name returns the name of the plugin -func (p *McoPlugin) Name() string { - return p.PluginName -} - -// Spec returns the version of the spec expected by the plugin -func (p *McoPlugin) Spec() string { - return p.SpecVersion -} - -// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateAdd()") - nodeName = state.GetName() - return p.OnNodeStateChange(nil, state) -} - -// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateChange()") - switchdevConfigured = utils.IsSwitchdevModeSpec(new.Spec) - - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) - return - } - if remove { - needDrain = true - return - } - if update { - if _, err = os.Stat(switchdevUnitPath); err != nil { - if os.IsNotExist(err) { - glog.Info("mco-plugin OnNodeStateChange(): the latest MachineConfig has not been applied") - needDrain = true - err = nil - return - } - return - } - // node is already in the offload MCP - glog.Info("mco-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true - return - } - return -} - -// Apply config change -func (p *McoPlugin) Apply() error { - glog.Info("mco-plugin Apply()") - node, err := kubeclient.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - label := nodeLabelPrefix + controllers.HwOffloadNodeLabel - if switchdevConfigured { - if _, ok := node.Labels[label]; !ok { - glog.Infof("Move node %s into HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: "", - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is already in HW offload MachineConfigPool", node.Name) - return nil - } - - if _, ok := node.Labels[label]; ok { - glog.Infof("Remove node %s from HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: nil, - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) - return nil -} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go index 26d7819a0..c94373178 100644 --- a/pkg/utils/switchdev.go +++ b/pkg/utils/switchdev.go @@ -23,7 +23,7 @@ func IsSwitchdevModeSpec(spec sriovnetworkv1.SriovNetworkNodeStateSpec) bool { return false } -func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update bool, err error) { _, err = os.Stat(switchDevConfPath) if err != nil { if os.IsNotExist(err) { @@ -52,10 +52,6 @@ func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (upd glog.V(2).Info("WriteSwitchdevConfFile(): no update") return } - if newContent == "" { - remove = true - glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") - } update = true glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0644)