Run sriov-cni containers in config daemon

When ovn hardware offload is enabled with ovn-k8s as default
CNI plugin, pods using veth as default interface type can no
longer be created since OpenvSwitch is offloaded to the
SmartNIC system (with current design). sriov-cni pod is affected
in such case. This commit moves sriov-cni into config daemonset
which runs in host network and doesn't use veth interface.

This also remove the sriov-cni daemonset created by Operator.

bindata/manifests/daemon/daemonset.yaml

@@ -32,6 +32,28 @@ spec:
       serviceAccountName: sriov-network-config-daemon
       priorityClassName: "system-node-critical"
       containers:
+      - name: sriov-cni
+        image: {{.SRIOVCNIImage}}
+        securityContext:
+          privileged: true
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+        volumeMounts:
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+      - name: sriov-infiniband-cni
+        image: {{.SRIOVInfiniBandCNIImage}}
+        securityContext:
+          privileged: true
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+        volumeMounts:
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
       - name: sriov-network-config-daemon
         image: {{.Image}}
         command:
@@ -67,3 +89,6 @@ spec:
       - name: host
         hostPath:
           path: /
+      - name: cnibin
+        hostPath:
+          path: {{.CNIBinPath}}

bindata/manifests/plugins/002-rbac.yaml

@@ -1,12 +1,6 @@
 ---
 apiVersion: v1
 kind: ServiceAccount
-metadata:
-  name: sriov-cni
-  namespace: {{.Namespace}}
----
-apiVersion: v1
-kind: ServiceAccount
 metadata:
   name: sriov-device-plugin
   namespace: {{.Namespace}}
@@ -28,20 +22,6 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
-metadata:
-  name: sriov-cni
-  namespace: {{.Namespace}}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: sriov-plugin
-subjects:
-  - kind: ServiceAccount
-    name: sriov-cni
-    namespace: {{.Namespace}}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
 metadata:
   name: sriov-device-plugin
   namespace: {{.Namespace}}

controllers/sriovnetworknodepolicy_controller.go

@@ -29,6 +29,7 @@ import (
 	errs "github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -315,21 +316,12 @@ func (r *SriovNetworkNodePolicyReconciler) syncPluginDaemonObjs(dp *sriovnetwork
 	logger := r.Log.WithName("syncPluginDaemonObjs")
 	logger.Info("Start to sync sriov daemons objects")
 
-	// render RawCNIConfig manifests
+	// render plugin manifests
 	data := render.MakeRenderData()
 	data.Data["Namespace"] = namespace
-	data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")
-	data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")
 	data.Data["SRIOVDevicePluginImage"] = os.Getenv("SRIOV_DEVICE_PLUGIN_IMAGE")
 	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
 	data.Data["ResourcePrefix"] = os.Getenv("RESOURCE_PREFIX")
-	envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH")
-	if envCniBinPath == "" {
-		data.Data["CNIBinPath"] = "/var/lib/cni/bin"
-	} else {
-		logger.Info("New cni bin found", "CNIBinPath", envCniBinPath)
-		data.Data["CNIBinPath"] = envCniBinPath
-	}
 
 	objs, err := renderDsForCR(PLUGIN_PATH, &data)
 	if err != nil {
@@ -377,6 +369,58 @@ func (r *SriovNetworkNodePolicyReconciler) syncPluginDaemonObjs(dp *sriovnetwork
 			return err
 		}
 	}
+
+	// Sriov-cni container has been moved to sriov-network-config-daemon DaemonSet.
+	// Delete stale sriov-cni manifests. Revert this change once sriov-cni daemonSet
+	// is deprecated.
+	err = r.deleteSriovCniManifests()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *SriovNetworkNodePolicyReconciler) deleteSriovCniManifests() error {
+	ds := &appsv1.DaemonSet{}
+	err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, ds)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	} else {
+		err = r.Delete(context.TODO(), ds)
+		if err != nil {
+			return err
+		}
+	}
+
+	rb := &rbacv1.RoleBinding{}
+	err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, rb)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	} else {
+		err = r.Delete(context.TODO(), rb)
+		if err != nil {
+			return err
+		}
+	}
+
+	sa := &corev1.ServiceAccount{}
+	err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "sriov-cni"}, sa)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	} else {
+		err = r.Delete(context.TODO(), sa)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

controllers/sriovoperatorconfig_controller.go

@@ -167,8 +167,17 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(dc *sriovnetworkv1.S
 	data := render.MakeRenderData()
 	data.Data["Image"] = os.Getenv("SRIOV_NETWORK_CONFIG_DAEMON_IMAGE")
 	data.Data["Namespace"] = namespace
+	data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")
+	data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")
 	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
 	data.Data["ClusterType"] = utils.ClusterType
+	envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH")
+	if envCniBinPath == "" {
+		data.Data["CNIBinPath"] = "/var/lib/cni/bin"
+	} else {
+		logger.Info("New cni bin found", "CNIBinPath", envCniBinPath)
+		data.Data["CNIBinPath"] = envCniBinPath
+	}
 	objs, err := render.RenderDir(CONFIG_DAEMON_PATH, &data)
 	if err != nil {
 		logger.Error(err, "Fail to render config daemon manifests")