Test splitting smoke tests by dependency (#5803)

.github/actions/smoke-tests/action.yaml

@@ -25,14 +25,20 @@ inputs:
   marker:
     description: Marker to use
     required: false
+  label:
+    description: Label for test
+    required: false
   azure-ad-secret:
     description: Azure Active Directory secret for JWKs
     required: false
 
 outputs:
   test-results-name:
     description: Test results name
-    value: tests-${{ steps.k8s.outputs.cluster }}
+    value: ${{ steps.k8s.outputs.test_name }}
+  test-results-path:
+    description: Test results full path
+    value: ${{ steps.k8s.outputs.test_output_path }}
 
 runs:
   using: composite
@@ -42,12 +48,20 @@ runs:
       run: |
         make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }}
         make -f tests/Makefile image-load REGISTRY="" PREFIX=${{ inputs.image-name }} TAG=${{ inputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
-        marker="${{ inputs.marker }}"
-        nospaces="${marker// /_}"
-        sanitized_marker="${nospaces//\'/}"
-        name="${sanitized_marker:-${{ inputs.k8s-version }}}"
-        echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT
-        echo "cluster=$(echo nginx-${{ inputs.image-type }}-$name)" >> $GITHUB_OUTPUT
+        label="${{ inputs.label }}"
+        nospaces="${label// /_}"
+        noslash="${nospaces//\//_}"
+        sanitized_marker="${noslash//\'/}"
+        name="${sanitized_marker}-${{ inputs.k8s-version }}"
+        cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)
+        test_name=tests-nginx-${{ inputs.image-type }}-${name}.html
+        test_output_path=${{ github.workspace }}/tests/${test_name}
+        echo "cluster_ip=${cluster_ip}" >> $GITHUB_OUTPUT
+        echo "test_name=${test_name}" >> $GITHUB_OUTPUT
+        echo "test_output_path=${test_output_path}" >> $GITHUB_OUTPUT
+        echo "Output:"
+        echo "  cluster_ip=${cluster_ip}"
+        echo "  test_output_path=${test_output_path}"
       shell: bash
 
     - name: Setup Kubeconfig
@@ -56,21 +70,23 @@ runs:
       shell: bash
 
     - name: Run Smoke Tests
+      id: smoke-tests
       run: |
-        touch tests-${{ steps.k8s.outputs.cluster }}.html
+        touch ${{ steps.k8s.outputs.test_output_path }}
         docker run --rm \
         --name test-runner-${{ github.run_id }} \
         --network=kind \
         -v ${{ github.workspace }}/tests:/workspace/tests \
         -v ${{ github.workspace }}/deployments:/workspace/deployments \
         -v ${{ github.workspace }}/config:/workspace/config \
+        -v ${{ steps.k8s.outputs.test_output_path }}:${{ steps.k8s.outputs.test_output_path }} \
         -v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
         --context=kind-${{ github.run_id }} \
         --image=${{ inputs.image-name }}:${{ inputs.tag }} \
         --image-pull-policy=Never \
         --ic-type=nginx${{ contains(inputs.image-type, 'plus') && '-plus' || '' }}-ingress \
         --service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \
-        --html=tests-${{ steps.k8s.outputs.cluster }}.html \
+        --html=${{ steps.k8s.outputs.test_output_path }} \
         --self-contained-html \
         --durations=10 \
         --show-ic-logs=yes \

.github/data/matrix-smoke-nap.json

@@ -0,0 +1,53 @@
+{
+  "images": [
+    {
+      "label": "AP_WAF 1/4",
+      "image": "ubi-9-plus-nap",
+      "type": "plus",
+      "nap_modules": "waf",
+      "marker": "appprotect_waf_policies_allow",
+      "platforms": "linux/amd64"
+    },
+    {
+      "label": "AP_WAF 2/4",
+      "image": "alpine-plus-nap-fips",
+      "type": "plus",
+      "nap_modules": "waf",
+      "marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'",
+      "platforms": "linux/amd64"
+    },
+    {
+      "label": "AP_WAF 3/4",
+      "image": "debian-plus-nap",
+      "type": "plus",
+      "nap_modules": "waf",
+      "marker": "appprotect_waf_policies_grpc",
+      "platforms": "linux/amd64"
+    },
+    {
+      "label": "AP_WAF 4/4",
+      "image": "debian-plus-nap",
+      "type": "plus",
+      "nap_modules": "waf",
+      "marker": "'appprotect_watch or appprotect_batch or appprotect_integration'",
+      "platforms": "linux/amd64"
+    },
+    {
+      "label": "AP_DOS",
+      "image": "debian-plus-nap",
+      "type": "plus",
+      "nap_modules": "dos",
+      "marker": "'dos and not dos_learning'",
+      "platforms": "linux/amd64"
+    },
+    {
+      "label": "AP_DOS_Learning",
+      "image": "ubi-8-plus-nap",
+      "type": "plus",
+      "nap_modules": "dos",
+      "marker": "dos_learning",
+      "platforms": "linux/amd64"
+    }
+  ],
+  "k8s": []
+}

.github/data/matrix-smoke-oss.json

@@ -0,0 +1,68 @@
+{
+  "images": [
+    {
+      "label": "ingresses 1/2",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "ingresses 2/2",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VSR",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "vsr",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "policies 1/2",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "policies 2/2",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 1/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 2/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 3/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "TS",
+      "image": "ubi",
+      "type": "oss",
+      "marker": "ts",
+      "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    }
+  ],
+  "k8s": []
+}

.github/data/matrix-smoke-plus.json

@@ -0,0 +1,68 @@
+{
+  "images": [
+    {
+      "label": "VS 1/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 2/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 3/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "TS",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "ts",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 1/2",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 2/2",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VSR",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "vsr",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "policies 1/2",
+      "image": "ubi-plus",
+      "type": "plus",
+      "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'",
+      "platforms": "linux/arm64, linux/amd64, linux/s390x"
+    },
+    {
+      "label": "policies 2/2",
+      "image": "ubi-plus",
+      "type": "plus",
+      "marker": "'policies_ac or policies_jwt or policies_mtls'",
+      "platforms": "linux/arm64, linux/amd64, linux/s390x"
+    }
+  ],
+  "k8s": []
+}