From a7ba933cf37e6c9eefab703c5a7f8a6fcc3b2d7f Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Fri, 25 Jun 2021 14:35:41 +0800 Subject: [PATCH] Run sriov-cni containers in config daemon When ovn hardware offload is enabled with ovn-k8s as default CNI plugin, pods using veth as default interface type can no longer be created since OpenvSwitch is offloaded to the SmartNIC system (with current design). sriov-cni pod is affected in such case. This commit moves sriov-cni into config daemonset which runs in host network and doesn't use veth interface. This also remove the sriov-cni daemonset created by Operator. --- bindata/manifests/daemon/daemonset.yaml | 25 ++++++++ bindata/manifests/plugins/002-rbac.yaml | 20 ------ bindata/manifests/plugins/sriov-cni.yaml | 60 ----------------- .../sriovnetworknodepolicy_controller.go | 64 ++++++++++++++++--- controllers/sriovoperatorconfig_controller.go | 9 +++ 5 files changed, 88 insertions(+), 90 deletions(-) delete mode 100644 bindata/manifests/plugins/sriov-cni.yaml diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index 81db900f4..6221d9b2a 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -32,6 +32,28 @@ spec: serviceAccountName: sriov-network-config-daemon priorityClassName: "system-node-critical" containers: + - name: sriov-cni + image: {{.SRIOVCNIImage}} + securityContext: + privileged: true + resources: + requests: + cpu: 10m + memory: 10Mi + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + - name: sriov-infiniband-cni + image: {{.SRIOVInfiniBandCNIImage}} + securityContext: + privileged: true + resources: + requests: + cpu: 10m + memory: 10Mi + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin - name: sriov-network-config-daemon image: {{.Image}} command: @@ -67,3 +89,6 @@ spec: - name: host hostPath: path: / + - name: cnibin + hostPath: + path: {{.CNIBinPath}} diff --git a/bindata/manifests/plugins/002-rbac.yaml b/bindata/manifests/plugins/002-rbac.yaml index f1b205b5e..f4892acf8 100644 --- a/bindata/manifests/plugins/002-rbac.yaml +++ b/bindata/manifests/plugins/002-rbac.yaml @@ -1,12 +1,6 @@ --- apiVersion: v1 kind: ServiceAccount -metadata: - name: sriov-cni - namespace: {{.Namespace}} ---- -apiVersion: v1 -kind: ServiceAccount metadata: name: sriov-device-plugin namespace: {{.Namespace}} @@ -28,20 +22,6 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding -metadata: - name: sriov-cni - namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sriov-plugin -subjects: - - kind: ServiceAccount - name: sriov-cni - namespace: {{.Namespace}} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding metadata: name: sriov-device-plugin namespace: {{.Namespace}} diff --git a/bindata/manifests/plugins/sriov-cni.yaml b/bindata/manifests/plugins/sriov-cni.yaml deleted file mode 100644 index d5111cae7..000000000 --- a/bindata/manifests/plugins/sriov-cni.yaml +++ /dev/null @@ -1,60 +0,0 @@ ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: sriov-cni - namespace: {{.Namespace}} - annotations: - kubernetes.io/description: | - This daemonset copies the SR-IOV CNI plugin on to each node. - release.openshift.io/version: "{{.ReleaseVersion}}" -spec: - selector: - matchLabels: - app: sriov-cni - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 33% - template: - metadata: - labels: - app: sriov-cni - component: network - type: infra - openshift.io/component: network - spec: - nodeSelector: - beta.kubernetes.io/os: linux - node-role.kubernetes.io/worker: - tolerations: - - operator: Exists - serviceAccountName: sriov-cni - priorityClassName: "system-node-critical" - containers: - - name: sriov-cni - image: {{.SRIOVCNIImage}} - securityContext: - privileged: true - resources: - requests: - cpu: 10m - memory: 10Mi - volumeMounts: - - name: cnibin - mountPath: /host/opt/cni/bin - - name: sriov-infiniband-cni - image: {{.SRIOVInfiniBandCNIImage}} - securityContext: - privileged: true - resources: - requests: - cpu: 10m - memory: 10Mi - volumeMounts: - - name: cnibin - mountPath: /host/opt/cni/bin - volumes: - - name: cnibin - hostPath: - path: {{.CNIBinPath}} diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index b6b7f82c8..239fce894 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -29,6 +29,7 @@ import ( errs "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -315,21 +316,12 @@ func (r *SriovNetworkNodePolicyReconciler) syncPluginDaemonObjs(dp *sriovnetwork logger := r.Log.WithName("syncPluginDaemonObjs") logger.Info("Start to sync sriov daemons objects") - // render RawCNIConfig manifests + // render plugin manifests data := render.MakeRenderData() data.Data["Namespace"] = namespace - data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE") - data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE") data.Data["SRIOVDevicePluginImage"] = os.Getenv("SRIOV_DEVICE_PLUGIN_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") data.Data["ResourcePrefix"] = os.Getenv("RESOURCE_PREFIX") - envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH") - if envCniBinPath == "" { - data.Data["CNIBinPath"] = "/var/lib/cni/bin" - } else { - logger.Info("New cni bin found", "CNIBinPath", envCniBinPath) - data.Data["CNIBinPath"] = envCniBinPath - } objs, err := renderDsForCR(PLUGIN_PATH, &data) if err != nil { @@ -377,6 +369,58 @@ func (r *SriovNetworkNodePolicyReconciler) syncPluginDaemonObjs(dp *sriovnetwork return err } } + + // Sriov-cni container has been moved to sriov-network-config-daemon DaemonSet. + // Delete stale sriov-cni manifests. Revert this change once sriov-cni daemonSet + // is deprecated. + err = r.deleteSriovCniManifests() + if err != nil { + return err + } + + return nil +} + +func (r *SriovNetworkNodePolicyReconciler) deleteSriovCniManifests() error { + ds := &appsv1.DaemonSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, ds) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + } else { + err = r.Delete(context.TODO(), ds) + if err != nil { + return err + } + } + + rb := &rbacv1.RoleBinding{} + err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, rb) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + } else { + err = r.Delete(context.TODO(), rb) + if err != nil { + return err + } + } + + sa := &corev1.ServiceAccount{} + err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, sa) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + } else { + err = r.Delete(context.TODO(), sa) + if err != nil { + return err + } + } + return nil } diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 5e3591912..baa7f52b9 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -167,8 +167,17 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(dc *sriovnetworkv1.S data := render.MakeRenderData() data.Data["Image"] = os.Getenv("SRIOV_NETWORK_CONFIG_DAEMON_IMAGE") data.Data["Namespace"] = namespace + data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE") + data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") data.Data["ClusterType"] = utils.ClusterType + envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH") + if envCniBinPath == "" { + data.Data["CNIBinPath"] = "/var/lib/cni/bin" + } else { + logger.Info("New cni bin found", "CNIBinPath", envCniBinPath) + data.Data["CNIBinPath"] = envCniBinPath + } objs, err := render.RenderDir(CONFIG_DAEMON_PATH, &data) if err != nil { logger.Error(err, "Fail to render config daemon manifests")