From 8ea792e3cfa47bb84dd182fa7d7acd0abe964620 Mon Sep 17 00:00:00 2001 From: adrianc Date: Wed, 19 Jun 2024 15:35:36 +0300 Subject: [PATCH 1/2] bump multus version - deploy multus thick plugin. update manifests accordingly - use sha256 digest as there is no recent release version of multus for now. Signed-off-by: adrianc --- deployment/network-operator/values.yaml | 4 +- ...f.io_networkattachmentdefinitions_crd.yaml | 2 +- hack/release.yaml | 4 +- .../state-multus-cni/0010-cluter_role.yml | 2 + manifests/state-multus-cni/0040-configmap.yml | 18 +++- manifests/state-multus-cni/0050-multus-ds.yml | 97 +++++++++++++++---- pkg/state/state_multus_cni_test.go | 65 ++++--------- 7 files changed, 119 insertions(+), 73 deletions(-) diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml index a7510a55f..388ecbf03 100644 --- a/deployment/network-operator/values.yaml +++ b/deployment/network-operator/values.yaml @@ -362,9 +362,9 @@ secondaryNetwork: # memory: "50Mi" multus: deploy: true - image: multus-cni + image: multus-cni@sha256 repository: ghcr.io/k8snetworkplumbingwg - version: v3.9.3 + version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8 # imagePullSecrets: [] # config: '' # containerResources: diff --git a/hack/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml b/hack/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml index 5916055f0..163097b3e 100644 --- a/hack/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml +++ b/hack/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml @@ -23,7 +23,7 @@ spec: singular: network-attachment-definition kind: NetworkAttachmentDefinition shortNames: - - net-attach-def + - net-attach-def versions: - name: v1 served: true diff --git a/hack/release.yaml b/hack/release.yaml index 5108ccf05..cdf6616a8 100644 --- a/hack/release.yaml +++ b/hack/release.yaml @@ -46,9 +46,9 @@ CniPlugins: repository: ghcr.io/k8snetworkplumbingwg version: v1.5.0 Multus: - image: multus-cni + image: multus-cni@sha256 repository: ghcr.io/k8snetworkplumbingwg - version: v3.9.3 + version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8 Ipoib: image: ipoib-cni repository: ghcr.io/mellanox diff --git a/manifests/state-multus-cni/0010-cluter_role.yml b/manifests/state-multus-cni/0010-cluter_role.yml index 155d378fe..da7e73269 100644 --- a/manifests/state-multus-cni/0010-cluter_role.yml +++ b/manifests/state-multus-cni/0010-cluter_role.yml @@ -34,6 +34,8 @@ rules: - pods/status verbs: - get + - list + - watch - update - apiGroups: - "" diff --git a/manifests/state-multus-cni/0040-configmap.yml b/manifests/state-multus-cni/0040-configmap.yml index ed9f3db2b..bcbdb5f81 100644 --- a/manifests/state-multus-cni/0040-configmap.yml +++ b/manifests/state-multus-cni/0040-configmap.yml @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -{{ if .CrSpec.Config -}} kind: ConfigMap apiVersion: v1 metadata: @@ -21,5 +20,18 @@ metadata: tier: node app: multus data: - cni-conf.json: '{{ .CrSpec.Config }}' -{{ end -}} +{{- if .CrSpec.Config }} + daemon-config.json: '{{ .CrSpec.Config }}' +{{- else }} + daemon-config.json: | + { + "chrootDir": "/hostroot", + "cniVersion": "0.3.1", + "logLevel": "verbose", + "logToStderr": true, + "cniConfigDir": "/host/etc/cni/net.d", + "multusAutoconfigDir": "/host/etc/cni/net.d", + "multusConfigFile": "auto", + "socketDir": "/host/run/multus/" + } +{{- end }} diff --git a/manifests/state-multus-cni/0050-multus-ds.yml b/manifests/state-multus-cni/0050-multus-ds.yml index 7cdbb29a0..00cbd45b1 100644 --- a/manifests/state-multus-cni/0050-multus-ds.yml +++ b/manifests/state-multus-cni/0050-multus-ds.yml @@ -19,8 +19,13 @@ spec: tier: node app: multus name: multus + annotations: + cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false" spec: hostNetwork: true + hostPID: true + priorityClassName: "system-node-critical" + terminationGracePeriodSeconds: 10 {{- if .NodeAffinity }} affinity: nodeAffinity: @@ -40,21 +45,31 @@ spec: - key: nvidia.com/gpu operator: Exists effect: NoSchedule + initContainers: + - name: install-multus-binary + image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }} + command: + - "cp" + - "-f" + - "/usr/src/multus-cni/bin/multus-shim" + - "/host/opt/cni/bin/multus-shim" + resources: + requests: + cpu: "10m" + memory: "15Mi" + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + mountPropagation: Bidirectional containers: - name: kube-multus image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }} - command: ["/entrypoint.sh"] + command: ["/usr/src/multus-cni/bin/multus-daemon"] args: - - "--cni-version=0.3.1" - # /tmp/multus-conf/00-multus.conf is where multus-cfg ConfigMap is mounted then entrypoint.sh copy it to - # /host/etc/cni/net.d/00-multus.conf - - "--multus-conf-file={{- if .CrSpec.Config -}}/tmp/multus-conf/00-multus.conf{{- else -}}auto{{- end -}}" - # Remove multus config file to prevent failing of creating/deleting pods since multus will fail due to - # permission issue, https://github.com/intel/multus-cni/issues/592 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -f /host/etc/cni/net.d/00-multus.conf"] + - "--config=/etc/cni/net.d/multus.d/daemon-config.json" {{- with .RuntimeSpec.ContainerResources }} {{- with index . "kube-multus" }} resources: @@ -78,15 +93,38 @@ spec: {{- end }} securityContext: privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cni mountPath: /host/etc/cni/net.d + # multus-daemon expects that cnibin path must be identical between pod and container host. + # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon, + # not to any other directory, like '/opt/bin' or '/usr/bin'. - name: cnibin - mountPath: /host/opt/cni/bin - {{- if .CrSpec.Config }} - - name: multus-cfg - mountPath: /tmp/multus-conf - {{- end }} + mountPath: /opt/cni/bin + - name: host-run + mountPath: /host/run + - name: host-var-lib-cni-multus + mountPath: /var/lib/cni/multus + - name: host-var-lib-kubelet + mountPath: /var/lib/kubelet + mountPropagation: HostToContainer + - name: host-run-k8s-cni-cncf-io + mountPath: /run/k8s.cni.cncf.io + - name: host-run-netns + mountPath: /run/netns + mountPropagation: HostToContainer + - name: multus-cni-config + mountPath: /etc/cni/net.d/multus.d + readOnly: true + - name: hostroot + mountPath: /hostroot + mountPropagation: HostToContainer + env: + - name: MULTUS_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumes: - name: cni hostPath: @@ -94,11 +132,28 @@ spec: - name: cnibin hostPath: path: {{ .RuntimeSpec.CniBinDirectory }} - {{- if .CrSpec.Config }} - - name: multus-cfg + - name: hostroot + hostPath: + path: / + - name: multus-cni-config configMap: name: multus-cni-config items: - - key: cni-conf.json - path: 00-multus.conf - {{- end }} + - key: daemon-config.json + path: daemon-config.json + - name: host-run + hostPath: + path: /run + - name: host-var-lib-cni-multus + hostPath: + path: /var/lib/cni/multus + - name: host-var-lib-kubelet + hostPath: + path: /var/lib/kubelet + - name: host-run-k8s-cni-cncf-io + hostPath: + path: /run/k8s.cni.cncf.io + - name: host-run-netns + hostPath: + path: /run/netns/ + diff --git a/pkg/state/state_multus_cni_test.go b/pkg/state/state_multus_cni_test.go index 40bd3c506..3ec8a66da 100644 --- a/pkg/state/state_multus_cni_test.go +++ b/pkg/state/state_multus_cni_test.go @@ -18,6 +18,7 @@ package state import ( "context" + "encoding/json" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -256,12 +257,8 @@ var _ = Describe("Multus CNI state", func() { })).To(BeTrue()) }) - It("should render resources correctly when config is specified in CR", func() { + It("should render config map with expected key", func() { cr := getMinimalNicClusterPolicyWithMultus() - - configString := "myconfig" - cr.Spec.SecondaryNetwork.Multus.Config = &configString - objs, err := state.GetManifestObjects(context.TODO(), cr, catalog, testLogger) Expect(err).NotTo(HaveOccurred()) @@ -271,53 +268,33 @@ var _ = Describe("Multus CNI state", func() { Expect(err).NotTo(HaveOccurred()) Expect(configMap.Namespace).To(Equal(networkOperatorResourceNamespace)) - Expect(configMap.Data["cni-conf.json"]).To(Equal(configString)) - })).To(BeTrue()) - - Expect(runFuncForObjectInSlice(objs, "DaemonSet", func(obj *unstructured.Unstructured) { - var daemonSet appsv1.DaemonSet - err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &daemonSet) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement( - corev1.VolumeMount{ - Name: "multus-cfg", - MountPath: "/tmp/multus-conf", - }, - )) - - Expect(daemonSet.Spec.Template.Spec.Volumes).To(ContainElement( - corev1.Volume{ - Name: "multus-cfg", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "multus-cni-config", - }, - Items: []corev1.KeyToPath{ - { - Key: "cni-conf.json", - Path: "00-multus.conf", - }, - }, - }, - }, - }, - )) - + defaultConfig := make(map[string]interface{}) + Expect(configMap.Data).To(HaveKey("daemon-config.json")) + // just make sure data is valid json + Expect(json.Unmarshal([]byte(configMap.Data["daemon-config.json"]), &defaultConfig)).ToNot(HaveOccurred()) + Expect(defaultConfig).ToNot(BeEmpty()) })).To(BeTrue()) }) - It("should not render ConfigMap if config is not specified in CR", func() { + It("should render config map with config as specified in CR", func() { cr := getMinimalNicClusterPolicyWithMultus() + + configString := "myconfig" + cr.Spec.SecondaryNetwork.Multus.Config = &configString + objs, err := state.GetManifestObjects(context.TODO(), cr, catalog, testLogger) Expect(err).NotTo(HaveOccurred()) - for _, obj := range objs { - Expect(obj.GetKind()).ToNot(Equal("ConfigMap")) - } - }) + Expect(runFuncForObjectInSlice(objs, "ConfigMap", func(obj *unstructured.Unstructured) { + var configMap corev1.ConfigMap + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &configMap) + Expect(err).NotTo(HaveOccurred()) + Expect(configMap.Namespace).To(Equal(networkOperatorResourceNamespace)) + Expect(configMap.Data).To(HaveKey("daemon-config.json")) + Expect(configMap.Data["daemon-config.json"]).To(Equal(configString)) + })).To(BeTrue()) + }) }) func getMinimalNicClusterPolicyWithMultus() *mellanoxv1alpha1.NicClusterPolicy { From a58ed833ef97639ad00693949dbc4f7bdd4bbb75 Mon Sep 17 00:00:00 2001 From: adrianc Date: Wed, 19 Jun 2024 15:38:09 +0300 Subject: [PATCH 2/2] run make release-build Signed-off-by: adrianc --- .../crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml | 4 ++-- ...mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml | 4 ++-- example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml index f1cadac59..406bb4254 100644 --- a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml +++ b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml @@ -66,9 +66,9 @@ spec: repository: ghcr.io/mellanox version: v1.2.0 multus: - image: multus-cni + image: multus-cni@sha256 repository: ghcr.io/k8snetworkplumbingwg - version: v3.9.3 + version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8 config: '' ipamPlugin: image: whereabouts diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml index e31d552f7..83cd8ebdb 100644 --- a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml +++ b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml @@ -62,9 +62,9 @@ spec: repository: ghcr.io/k8snetworkplumbingwg version: v1.5.0 multus: - image: multus-cni + image: multus-cni@sha256 repository: ghcr.io/k8snetworkplumbingwg - version: v3.9.3 + version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8 config: '' nvIpam: image: nvidia-k8s-ipam diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml index dba93bb0c..ea4dec09b 100644 --- a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml +++ b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml @@ -62,9 +62,9 @@ spec: repository: ghcr.io/k8snetworkplumbingwg version: v1.5.0 multus: - image: multus-cni + image: multus-cni@sha256 repository: ghcr.io/k8snetworkplumbingwg - version: v3.9.3 + version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8 config: '' ipamPlugin: image: whereabouts