Add k8s PSS policy example (#9233)

Signed-off-by: Artem Glazychev <[email protected]>

apps/csi-driver/csi-driver.yaml

@@ -0,0 +1,113 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: "csi.networkservicemesh.io"
+spec:
+  # Only ephemeral, inline volumes are supported. There is no need for a
+  # controller to provision and attach volumes.
+  attachRequired: false
+
+  # Request the pod information which the CSI driver uses to verify that an
+  # ephemeral mount was requested.
+  podInfoOnMount: true
+
+  # Don't change ownership on the contents of the mount since the
+  # NS registration Unix Domain Socket is typically open to all (i.e. 0777).
+  fsGroupPolicy: None
+
+  # Declare support for ephemeral volumes only.
+  volumeLifecycleModes:
+    - Ephemeral
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nsm-csi
+  labels:
+    app: nsm-csi
+spec:
+  selector:
+    matchLabels:
+      app: nsm-csi
+  template:
+    metadata:
+      labels:
+        app: nsm-csi
+    spec:
+      containers:
+        # This is the container which runs the NSM CSI driver.
+        - name: nsm-csi-driver
+          image: ghcr.io/networkservicemesh/ci/cmd-csi-driver:0d19b58
+          imagePullPolicy: IfNotPresent
+          env:
+            # The CSI driver needs a unique node ID. The node name can be
+            # used for this purpose.
+            - name: NSM_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: NSM_SOCKET_DIR
+              value: "/nsm-socket"
+            - name: NSM_CSI_SOCKET_PATH
+              value: "/nsm-csi/csi.sock"
+            - name: NSM_VERSION
+              value: cmd-csi-driver:0d19b58
+          volumeMounts:
+            # The volume containing the Network Service API socket.
+            # The NSM CSI driver will mount this directory into containers.
+            - mountPath: /nsm-socket
+              name: nsm-socket
+              readOnly: true
+            # The volume that will contain the CSI driver socket shared
+            # with the kubelet and the driver registrar.
+            - mountPath: /nsm-csi
+              name: nsm-csi-socket-dir
+            # The volume containing mount points for containers.
+            - mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+          securityContext:
+            privileged: true
+        # This container runs the CSI Node Driver Registrar which takes care
+        # of all the little details required to register a CSI driver with
+        # the kubelet.
+        - name: node-driver-registrar
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
+          imagePullPolicy: IfNotPresent
+          args: [
+            "-csi-address", "/nsm-csi/csi.sock",
+            "-kubelet-registration-path", "/var/lib/kubelet/plugins/csi.networkservicemesh.io/csi.sock",
+          ]
+          volumeMounts:
+            # The registrar needs access to the NSM CSI driver socket
+            - mountPath: /nsm-csi
+              name: nsm-csi-socket-dir
+            # The registrar needs access to the Kubelet plugin registration
+            # directory
+            - name: kubelet-plugin-registration-dir
+              mountPath: /registration
+      volumes:
+        - name: nsm-socket
+          hostPath:
+            path: /var/lib/networkservicemesh
+            type: DirectoryOrCreate
+        # This volume is where the socket for kubelet->driver communication lives
+        - name: nsm-csi-socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.networkservicemesh.io
+            type: DirectoryOrCreate
+        # This volume is where the NSM CSI driver mounts volumes
+        - name: mountpoint-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        # This volume is where the node-driver-registrar registers the plugin
+        # with kubelet
+        - name: kubelet-plugin-registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+        - name: exclude-prefixes-volume
+          emptyDir: {}

apps/csi-driver/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- csi-driver.yaml

examples/features/webhook/README.md

@@ -1,6 +1,6 @@
-# Client requests for postgresql service
+# Client requests for nginx service
 
-This example demonstrates how Postgres-client can get connectivity to Postgres-server deployment via NSM.
+This example demonstrates how the client can get connectivity to the nginx-server via NSM.
 Client pod and server deployment located on different nodes.
 
 

examples/pss/README.md

@@ -0,0 +1,45 @@
+# Pod Security Standard (PSS) examples
+
+Contain basic setup for NSM that includes `nsm-admission-webhook` `nsmgr`, `forwarder-vpp`, `registry-k8s` and [NSM CSI driver](https://github.com/networkservicemesh/cmd-csi-driver).
+CSI driver allows us to avoid using `hostPath` volumes in workloads.
+
+Based on the [PSS profile](https://kubernetes.io/docs/concepts/security/pod-security-standards/), the admission-webhook adds the required security settings to the NSM sidecar containers.
+
+**_Please note_** that the webhook only knows about the profile from **_the namespace labels_**.
+
+## Requires
+
+- [spire_csi](../spire/single_cluster_csi)
+
+## Includes
+
+- [Nginx service](use-cases/nginx)
+
+## Run
+
+Apply NSM resources:
+
+```bash
+kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/pss/nsm-system?ref=4e75403173ffc0f0fcf2cfa0692565ccf836c04c
+```
+
+Wait for admission-webhook-k8s:
+
+```bash
+WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+kubectl wait --for=condition=ready --timeout=1m pod ${WH} -n nsm-system
+```
+
+## Cleanup
+
+Due to CSI driver limitations, we first need to remove pods that contain a volume mounted by the driver:
+```bash
+kubectl delete ds/forwarder-vpp -n nsm-system
+```
+
+To free resources follow the next commands:
+```bash
+WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+kubectl delete mutatingwebhookconfiguration ${WH}
+kubectl delete ns nsm-system
+```

examples/pss/nsm-system/kustomization.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: nsm-system
+
+resources:
+  - nsm-system-namespace.yaml
+  - ../../../apps/csi-driver
+  - ../../../apps/nsmgr
+  - ../../../apps/forwarder-vpp
+  - ../../../apps/registry-k8s
+  - ../../../apps/admission-webhook-k8s
+
+patches:
+  - path: patch-nsmgr.yaml
+  - path: patch-forwarder-vpp.yaml
+  - path: patch-registry-k8s.yaml
+  - path: patch-admission-webhook-k8s.yaml

examples/pss/nsm-system/nsm-system-namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nsm-system

examples/pss/nsm-system/patch-admission-webhook-k8s.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: admission-webhook-k8s
+spec:
+  template:
+    spec:
+      containers:
+        - name: admission-webhook-k8s
+          env:
+            - name: NSM_ENVS
+              value: NSM_LOG_LEVEL=TRACE,NSM_LOCALDNSSERVERENABLED=False

examples/pss/nsm-system/patch-forwarder-vpp.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: forwarder-vpp
+spec:
+  template:
+    spec:
+      volumes:
+        - name: spire-agent-socket
+          hostPath: null
+          csi:
+            driver: "csi.spiffe.io"
+            readOnly: true
+        - name: nsm-socket
+          hostPath: null
+          csi:
+            driver: "csi.networkservicemesh.io"
+            readOnly: true

examples/pss/nsm-system/patch-nsmgr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nsmgr
+spec:
+  template:
+    spec:
+      volumes:
+        - name: spire-agent-socket
+          hostPath: null
+          csi:
+            driver: "csi.spiffe.io"
+            readOnly: true

examples/pss/nsm-system/patch-registry-k8s.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry-k8s
+spec:
+  template:
+    spec:
+      volumes:
+        - name: spire-agent-socket
+          hostPath: null
+          csi:
+            driver: "csi.spiffe.io"
+            readOnly: true

examples/pss/use-cases/nginx/README.md

@@ -0,0 +1,38 @@
+# Nginx service
+
+This example uses the `restricted` PSS policy for the namespace.
+The `restricted` policy requires additional `securityContext` settings as well as not using `hostPath`.
+
+We can see how the client can get connectivity to nginx-server via NSM.
+Client pod and server deployment located on different nodes.
+
+## Requires
+
+Make sure that you have completed steps from [PSS](../..).
+
+## Run
+
+Deploy client and nginx-nse
+```bash
+kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/pss/use-cases/nginx?ref=4e75403173ffc0f0fcf2cfa0692565ccf836c04c
+```
+
+Wait for applications ready:
+```bash
+kubectl wait --for=condition=ready --timeout=5m pod -l app=nse-kernel -n ns-nginx
+```
+```bash
+kubectl wait --for=condition=ready --timeout=1m pod -l app=nettools -n ns-nginx
+```
+
+Try to connect from client to nginx service:
+```bash
+kubectl exec pods/nettools -n ns-nginx -- curl 172.16.1.100:8080 | grep -o "<title>Welcome to nginx"
+```
+
+## Cleanup
+
+Delete ns:
+```bash
+kubectl delete ns ns-nginx
+```

examples/pss/use-cases/nginx/client.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nettools
+  labels:
+    app: nettools
+  annotations:
+    networkservicemesh.io: kernel://nginx/nsm-1
+spec:
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 10001
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: nettools
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop: ["ALL"]
+    image: travelping/nettools:1.10.1
+    imagePullPolicy: IfNotPresent
+    # simple `sleep` command would work
+    # but we need `trap` to be able to delete pods quckly
+    command: ["/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"]
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchExpressions:
+              - key: app
+                operator: In
+                values:
+                  - nse-kernel

examples/pss/use-cases/nginx/kustomization.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ns-nginx
+
+resources:
+- ns-nginx.yaml
+- client.yaml
+- netsvc.yaml
+- ../../../../apps/nse-kernel
+
+patches:
+  - path: patch-nse.yaml

examples/pss/use-cases/nginx/netsvc.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: networkservicemesh.io/v1
+kind: NetworkService
+metadata:
+  name: nginx
+spec:
+  payload: ETHERNET

examples/pss/use-cases/nginx/ns-nginx.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ns-nginx
+  labels:
+    pod-security.kubernetes.io/enforce: restricted