From 25d17eb9d6cae184596e39d86d01238172730509 Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Fri, 25 Jun 2021 14:35:41 +0800 Subject: [PATCH] Run sriov-cni containers in config daemon When ovn hardware offload is enabled with ovn-k8s as default CNI plugin, pods using veth as default interface type can no longer be created since OpenvSwitch is offloaded to the SmartNIC system (with current design). sriov-cni pod is affected in such case. This commit moves sriov-cni into config daemonset which runs in host network and doesn't use veth interface. This also remove the sriov-cni daemonset created by Operator. --- bindata/manifests/daemon/daemonset.yaml | 25 ++++++++ bindata/manifests/plugins/002-rbac.yaml | 20 ------- bindata/manifests/plugins/sriov-cni.yaml | 60 ------------------- .../sriovnetworknodepolicy_controller.go | 11 +--- controllers/sriovoperatorconfig_controller.go | 9 +++ 5 files changed, 35 insertions(+), 90 deletions(-) delete mode 100644 bindata/manifests/plugins/sriov-cni.yaml diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index 81db900f4c..6221d9b2a6 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -32,6 +32,28 @@ spec: serviceAccountName: sriov-network-config-daemon priorityClassName: "system-node-critical" containers: + - name: sriov-cni + image: {{.SRIOVCNIImage}} + securityContext: + privileged: true + resources: + requests: + cpu: 10m + memory: 10Mi + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + - name: sriov-infiniband-cni + image: {{.SRIOVInfiniBandCNIImage}} + securityContext: + privileged: true + resources: + requests: + cpu: 10m + memory: 10Mi + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin - name: sriov-network-config-daemon image: {{.Image}} command: @@ -67,3 +89,6 @@ spec: - name: host hostPath: path: / + - name: cnibin + hostPath: + path: {{.CNIBinPath}} diff --git a/bindata/manifests/plugins/002-rbac.yaml b/bindata/manifests/plugins/002-rbac.yaml index f1b205b5ed..f4892acf87 100644 --- a/bindata/manifests/plugins/002-rbac.yaml +++ b/bindata/manifests/plugins/002-rbac.yaml @@ -1,12 +1,6 @@ --- apiVersion: v1 kind: ServiceAccount -metadata: - name: sriov-cni - namespace: {{.Namespace}} ---- -apiVersion: v1 -kind: ServiceAccount metadata: name: sriov-device-plugin namespace: {{.Namespace}} @@ -28,20 +22,6 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding -metadata: - name: sriov-cni - namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sriov-plugin -subjects: - - kind: ServiceAccount - name: sriov-cni - namespace: {{.Namespace}} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding metadata: name: sriov-device-plugin namespace: {{.Namespace}} diff --git a/bindata/manifests/plugins/sriov-cni.yaml b/bindata/manifests/plugins/sriov-cni.yaml deleted file mode 100644 index d5111cae72..0000000000 --- a/bindata/manifests/plugins/sriov-cni.yaml +++ /dev/null @@ -1,60 +0,0 @@ ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: sriov-cni - namespace: {{.Namespace}} - annotations: - kubernetes.io/description: | - This daemonset copies the SR-IOV CNI plugin on to each node. - release.openshift.io/version: "{{.ReleaseVersion}}" -spec: - selector: - matchLabels: - app: sriov-cni - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 33% - template: - metadata: - labels: - app: sriov-cni - component: network - type: infra - openshift.io/component: network - spec: - nodeSelector: - beta.kubernetes.io/os: linux - node-role.kubernetes.io/worker: - tolerations: - - operator: Exists - serviceAccountName: sriov-cni - priorityClassName: "system-node-critical" - containers: - - name: sriov-cni - image: {{.SRIOVCNIImage}} - securityContext: - privileged: true - resources: - requests: - cpu: 10m - memory: 10Mi - volumeMounts: - - name: cnibin - mountPath: /host/opt/cni/bin - - name: sriov-infiniband-cni - image: {{.SRIOVInfiniBandCNIImage}} - securityContext: - privileged: true - resources: - requests: - cpu: 10m - memory: 10Mi - volumeMounts: - - name: cnibin - mountPath: /host/opt/cni/bin - volumes: - - name: cnibin - hostPath: - path: {{.CNIBinPath}} diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index b6b7f82c83..f18268e12b 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -315,21 +315,12 @@ func (r *SriovNetworkNodePolicyReconciler) syncPluginDaemonObjs(dp *sriovnetwork logger := r.Log.WithName("syncPluginDaemonObjs") logger.Info("Start to sync sriov daemons objects") - // render RawCNIConfig manifests + // render plugin manifests data := render.MakeRenderData() data.Data["Namespace"] = namespace - data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE") - data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE") data.Data["SRIOVDevicePluginImage"] = os.Getenv("SRIOV_DEVICE_PLUGIN_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") data.Data["ResourcePrefix"] = os.Getenv("RESOURCE_PREFIX") - envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH") - if envCniBinPath == "" { - data.Data["CNIBinPath"] = "/var/lib/cni/bin" - } else { - logger.Info("New cni bin found", "CNIBinPath", envCniBinPath) - data.Data["CNIBinPath"] = envCniBinPath - } objs, err := renderDsForCR(PLUGIN_PATH, &data) if err != nil { diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 5e35919126..baa7f52b9f 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -167,8 +167,17 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(dc *sriovnetworkv1.S data := render.MakeRenderData() data.Data["Image"] = os.Getenv("SRIOV_NETWORK_CONFIG_DAEMON_IMAGE") data.Data["Namespace"] = namespace + data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE") + data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") data.Data["ClusterType"] = utils.ClusterType + envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH") + if envCniBinPath == "" { + data.Data["CNIBinPath"] = "/var/lib/cni/bin" + } else { + logger.Info("New cni bin found", "CNIBinPath", envCniBinPath) + data.Data["CNIBinPath"] = envCniBinPath + } objs, err := render.RenderDir(CONFIG_DAEMON_PATH, &data) if err != nil { logger.Error(err, "Fail to render config daemon manifests")