Multus should request certificates per-node.

This introduces configurations for multus per-node certification.
openshift · Sep 21, 2023 · 76b1988 · 76b1988
1 parent 784c350
commit 76b1988
Show file tree

Hide file tree

Showing 10 changed files with 158 additions and 19 deletions.
diff --git a/bindata/network/multus-networkpolicy/multus-networkpolicy.yaml b/bindata/network/multus-networkpolicy/multus-networkpolicy.yaml
@@ -53,7 +53,7 @@ spec:
         kubernetes.io/os: linux
       priorityClassName: "system-node-critical"
       restartPolicy: Always
-      serviceAccountName: multus
+      serviceAccountName: multus-ancillary-tools
       tolerations:
       - operator: Exists
       volumes:

diff --git a/bindata/network/multus/002-rbac.yaml b/bindata/network/multus/002-rbac.yaml
@@ -1,8 +1,9 @@
 ---
+# This role is for Multus itself, hence the name "multus-proper"
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: multus
+  name: multus-proper
 rules:
 - apiGroups: ["apiextensions.k8s.io"]
   resources:
@@ -53,26 +54,95 @@ rules:
   - create
   - patch
   - update
+- apiGroups: ["certificates.k8s.io"]
+  resources:
+    - certificatesigningrequests
+  verbs:
+    - create
+    - get
+    - list
+    - watch
+
+---
+# This role is ancillary tools in the multus namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus-ancillary-tools
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups: ["", "events.k8s.io"]
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-group
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-proper
+subjects:
+  - kind: Group
+    name: system:multus
+    apiGroup: rbac.authorization.k8s.io
 
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
-
 ---
-apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: multus
+  name: multus-ancillary-tools
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: multus
+  name: multus-ancillary-tools
 subjects:
 - kind: ServiceAccount
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
 
 ---
@@ -83,7 +153,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: multus
+  name: multus-ancillary-tools
 subjects:
 - kind: Group
   name: system:cluster-readers
@@ -99,7 +169,7 @@ roleRef:
   name: whereabouts-cni
 subjects:
 - kind: ServiceAccount
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
 
 ---
@@ -114,7 +184,7 @@ roleRef:
   name: whereabouts-cni
 subjects:
 - kind: ServiceAccount
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
 
 ---

diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml
@@ -121,6 +121,11 @@ data:
         "logToStderr": true,
         "logLevel": "verbose",
         "binDir": "{{ .CNIBinDir }}",
+        "perNodeCertificate": {
+          "enabled": true,
+          "bootstrapKubeconfig": "/hostroot/var/lib/kubelet/kubeconfig",
+          "certDir": "/run/multus/certs"
+        },
         "cniConfigDir": "/host/etc/cni/net.d",
         "multusConfigFile": "auto",
         "multusAutoconfigDir": "/host/run/multus/cni/net.d",
@@ -166,10 +171,10 @@ spec:
       hostPID: true
       nodeSelector:
         kubernetes.io/os: linux
+      # serviceAccountName: multus-proper
       priorityClassName: "system-node-critical"
       tolerations:
       - operator: Exists
-      serviceAccountName: multus
       containers:
       - name: kube-multus
         image: {{.MultusImage}}
@@ -220,6 +225,8 @@ spec:
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
           readOnly: true
+        - name: host-run-multus-certs
+          mountPath: /run/multus/certs
         env:
         - name: RHEL8_SOURCE_DIRECTORY
           value: "/usr/src/multus-cni/rhel8/bin/"
@@ -243,6 +250,14 @@ spec:
         - name: "NO_PROXY"
           value: "{{ .NO_PROXY}}"
 {{ end }}
+        - name: K8S_NODE
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: MULTUS_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       terminationGracePeriodSeconds: 10
       volumes:
         - name: system-cni-dir
@@ -299,6 +314,9 @@ spec:
             items:
             - key: daemon-config.json
               path: daemon-config.json
+        - name: host-run-multus-certs
+          hostPath:
+            path: /run/multus_certs
 ---
 kind: DaemonSet
 apiVersion: apps/v1
@@ -333,7 +351,6 @@ spec:
       priorityClassName: "system-node-critical"
       tolerations:
       - operator: Exists
-      serviceAccountName: multus
       initContainers:
       - name: egress-router-binary-copy
         image: {{.EgressRouterImage}}
@@ -621,7 +638,7 @@ spec:
       hostNetwork: true      
       nodeSelector:
         kubernetes.io/os: linux
-      serviceAccountName: multus
+      serviceAccountName: multus-ancillary-tools
       tolerations:
       - operator: Exists
         effect: NoSchedule

diff --git a/...ntity/common/node-identity-namespace.yaml → ...y/common/001-node-identity-namespace.yaml b/...ntity/common/node-identity-namespace.yaml → ...y/common/001-node-identity-namespace.yaml
diff --git a/...e-identity/common/node-identity-rbac.yaml → ...entity/common/002-node-identity-rbac.yaml b/...e-identity/common/node-identity-rbac.yaml → ...entity/common/002-node-identity-rbac.yaml
diff --git a/bindata/network/node-identity/common/003-node-identity-configmap.yml b/bindata/network/node-identity/common/003-node-identity-configmap.yml
@@ -0,0 +1,22 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovnkube-identity-cm
+  namespace: openshift-network-node-identity
+  annotations:
+    kubernetes.io/description: |
+      This configmap contains the ovnkube-identity configuration files.
+data:
+  additional-cert-acceptance-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "organizations": ["system:multus"],
+      "groups": ["system:nodes", "system:multus", "system:authenticated"],
+      "userPrefixes": ["system:node", "system:multus"]
+    }]
+  additional-pod-admission-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "allowedPodAnnotations": ["k8s.v1.cni.cncf.io/network-status"]
+    }]
diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml
@@ -125,6 +125,7 @@ spec:
                 --enable-interconnect \
                 --disable-approver \
                 --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \
+                --pod-admission-conditions="/var/run/ovnkube-identity-config/additional-pod-admission-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -147,6 +148,8 @@ spec:
             name: hosted-cluster-api-access
           - mountPath: /hosted-ca
             name: hosted-ca-cert
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       - name: approver
         image: "{{.NetworkNodeIdentityImage}}"
         command:
@@ -173,6 +176,7 @@ spec:
             exec /usr/bin/ovnkube-identity \
                 --kubeconfig=/var/run/secrets/hosted_cluster/kubeconfig \
                 --lease-namespace=openshift-network-node-identity \
+                --csr-acceptance-conditions="/var/run/ovnkube-identity-config/additional-cert-acceptance-cond.json" \
                 --disable-webhook \
                 --loglevel="${LOGLEVEL}"
         env:
@@ -190,6 +194,8 @@ spec:
             name: hosted-cluster-api-access
           - mountPath: /hosted-ca
             name: hosted-ca-cert
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       # token-minter creates a token with the default service account path
       # The token is read by the containers to authenticate against the hosted cluster api server
       - name: token-minter
@@ -236,6 +242,14 @@ spec:
           secret:
             defaultMode: 0640
             secretName: network-node-identity-secret
+        - name: ovnkube-identity-cm
+          configMap:
+            name: ovnkube-identity-cm
+            items:
+              - key: additional-cert-acceptance-cond.json
+                path: additional-cert-acceptance-cond.json
+              - key: additional-pod-admission-cond.json
+                path: additional-pod-admission-cond.json
       tolerations:
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"

diff --git a/bindata/network/node-identity/self-hosted/node-identity.yaml b/bindata/network/node-identity/self-hosted/node-identity.yaml
@@ -61,6 +61,7 @@ spec:
                 --disable-approver \
                 --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \
                 --wait-for-kubernetes-api={{.NetworkNodeIdentityTerminationDurationSeconds}}s \
+                --pod-admission-conditions="/var/run/ovnkube-identity-config/additional-pod-admission-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -81,6 +82,8 @@ spec:
             name: audit-dir
           - mountPath: /env
             name: env-overrides
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       - name: approver
         image: "{{.NetworkNodeIdentityImage}}"
         command:
@@ -97,6 +100,7 @@ spec:
             echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start approver"
             exec /usr/bin/ovnkube-identity  --k8s-apiserver={{.K8S_APISERVER}} \
                 --disable-webhook \
+                --csr-acceptance-conditions="/var/run/ovnkube-identity-config/additional-cert-acceptance-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -109,6 +113,8 @@ spec:
         volumeMounts:
           - mountPath: /env
             name: env-overrides
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       terminationGracePeriodSeconds: {{.NetworkNodeIdentityTerminationDurationSeconds}}
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -124,6 +130,14 @@ spec:
         - hostPath:
             path: /var/log/kube-apiserver
           name: audit-dir
+        - name: ovnkube-identity-cm
+          configMap:
+            name: ovnkube-identity-cm
+            items:
+              - key: additional-cert-acceptance-cond.json
+                path: additional-cert-acceptance-cond.json
+              - key: additional-pod-admission-cond.json
+                path: additional-pod-admission-cond.json
       tolerations:
       - key: "node-role.kubernetes.io/master"
         operator: "Exists"

diff --git a/pkg/network/multus_test.go b/pkg/network/multus_test.go
@@ -49,11 +49,13 @@ func TestRenderMultus(t *testing.T) {
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "multus")))
 
 	// It's important that the namespace is first
-	g.Expect(len(objs)).To(Equal(26), "Expected 26 multus related objects")
+	g.Expect(len(objs)).To(Equal(28), "Expected 28 multus related objects")
 	g.Expect(objs[0]).To(HaveKubernetesID("CustomResourceDefinition", "", "network-attachment-definitions.k8s.cni.cncf.io"))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("Namespace", "", "openshift-multus")))
-	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus")))
-	g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceAccount", "openshift-multus", "multus")))
-	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceAccount", "openshift-multus", "multus-ancillary-tools")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus-ancillary-tools")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "multus-ancillary-tools")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus-proper")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "multus-group")))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "multus")))
 }
diff --git a/pkg/network/network_metrics_test.go b/pkg/network/network_metrics_test.go
@@ -49,7 +49,7 @@ func TestRenderNetworkMetricsDaemon(t *testing.T) {
 
 	// Check rendered object
 
-	g.Expect(len(objs)).To(Equal(26), "Expected 26 multus related objects")
+	g.Expect(len(objs)).To(Equal(28), "Expected 28 multus related objects")
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "network-metrics-daemon")))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("Service", "openshift-multus", "network-metrics-service")))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "metrics-daemon-role")))