Test CNI ordering

Followup to #242

This adds a new integration test that verifies that pods don't get
scheduled until all the networking has been setup, making sure having
linkerd-cni run run first doesn't trigger scheduling.

Note that for now we just consider the linkerd-cni+calico combo.

Worklow:

1. Install calico and linkerd-cni.
2. Label node and then add node selectors to calico and linkerd-cni to run only on the current node.
3. Create a new node. Calico nor linkerd-cni will be triggered in it.
4. Persist an nginx pod in that node. Verify it doesn't get scheduled.
5. Add label to linkerd-cni to have it trigger in the new node.
6. Verify the nginx pod still isn't scheduled.
7. Add label to calico to have it trigger in the new node.
8. Verify the nginx pod is finally scheduled.

Note that currently the test fails at step 6, but will pass as soon #242
gets merged. In order to test succesfully, you can apply this patch:

```diff
diff --git a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml
index 05f110b..419abdd 100644
--- a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml
+++ b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml
@@ -123,7 +123,7 @@ spec:
       # script copies the files into place and then sleeps so
       # that Kubernetes doesn't keep trying to restart it.
       - name: install-cni
-        image: test.l5d.io/linkerd/cni-plugin:test
+        image: ghcr.io/alpeb/cni-plugin:v1.2.8
         #image: cr.l5d.io/linkerd/cni-plugin:edge-22.12.1
         env:
         - name: DEST_CNI_NET_DIR
```

.github/workflows/cni-plugin-integration.yml

@@ -36,4 +36,13 @@ jobs:
       - uses: linkerd/dev/actions/setup-tools@v41
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
       - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-cilium
+        run: just cni-plugin-test-integration-cilium
+  ordering-test:
+    continue-on-error: true
+    timeout-minutes: 15
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: linkerd/dev/actions/setup-tools@v40
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
+      - name: Run CNI ordering tests
+        run: just cni-plugin-test-ordering

cni-plugin/integration/run-ordering.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "${BASH_SOURCE[0]%/*}"
+
+NODE_NAME=l5d-server-extra
+
+kubectl config use-context "k3d-$K3D_CLUSTER_NAME"
+
+printf '\n# Install calico and linkerd-cni...\n'
+kubectl apply -f manifests/calico/calico-install.yaml
+kubectl apply -f manifests/calico/linkerd-cni.yaml
+
+printf '\n# Label node and then add node selectors to calico and linkerd-cni to run only on the current node...\n'
+kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-calico=true
+kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-linkerd-cni=true
+kubectl -n kube-system patch daemonsets calico-node --type=json \
+	-p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-calico": "true"}}]'
+kubectl -n linkerd-cni patch daemonsets linkerd-cni --type=json \
+	-p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-linkerd-cni": "true"}}]'
+kubectl rollout status daemonset -n kube-system
+kubectl rollout status daemonset -n linkerd-cni
+
+printf '\n# Create new node...\n'
+k3d node create "$NODE_NAME" -c "$K3D_CLUSTER_NAME"
+k3d image import test.l5d.io/linkerd/cni-plugin:test -c "$K3D_CLUSTER_NAME"
+
+printf '\n# Start pod; k8s should not schedule it just yet...\n'
+selector="{
+  \"spec\": {
+    \"nodeSelector\": {
+      \"kubernetes.io/hostname\": \"k3d-$NODE_NAME-0\"
+    }
+  }
+}"
+kubectl run nginx --image nginx --restart Never --overrides="$selector"
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled as expected"
+else
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+
+printf '\n# Trigger linkerd-cni; k8s should still not schedule the pod...\n'
+kubectl label node "k3d-$NODE_NAME-0" allow-linkerd-cni=true
+kubectl rollout status daemonset -n linkerd-cni
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled as expected"
+else
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+
+printf '\n# Trigger calico; k8s should now schedule the pod...\n'
+kubectl label node "k3d-$NODE_NAME-0" allow-calico=true
+kubectl rollout status daemonset -n kube-system
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -c .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled unexpectedly"
+    exit1
+elif [[ $(echo "$status" | jq .state.running) == "null" ]]; then
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+echo 'Pod scheduled as expected.'

justfile

@@ -184,6 +184,14 @@ cni-plugin-test-integration-cilium:
         K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI_NO_POLICY_ENFORCER }}' \
         cni-plugin-test-integration-with-cilium
 
+cni-plugin-test-ordering: build-cni-plugin-image
+    @{{ just_executable() }} K3D_CLUSTER_NAME='l5d-calico-ordering-test' _cni-plugin-test-ordering-run
+
+_cni-plugin-test-ordering-run:
+    @{{ just_executable() }} K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI }}' _k3d-cni-create
+    @just-k3d import {{ cni-plugin-image }}
+    ./cni-plugin/integration/run-ordering.sh
+
 _cni-plugin-setup-cilium:
     #!/usr/bin/env bash
     set -euxo pipefail