From 202b1200edffdc275abdbefa567dcdb0eb1ec4bc Mon Sep 17 00:00:00 2001 From: Paul Abel <128620221+pdabelf5@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:03:23 +0000 Subject: [PATCH] Vs markers (#4793) split test matrix for vs and policies to reduce pipeline duration --- .github/workflows/ci.yml | 19 +++++++++++++------ pyproject.toml | 13 +++++++++++++ tests/suite/test_ac_policies.py | 1 + tests/suite/test_ac_policies_vsr.py | 1 + tests/suite/test_disable_ipv6.py | 1 + tests/suite/test_dos.py | 1 + tests/suite/test_egress_mtls.py | 1 + tests/suite/test_ingress_mtls.py | 2 ++ tests/suite/test_jwt_policies.py | 1 + tests/suite/test_jwt_policies_vsr.py | 1 + tests/suite/test_rewrites.py | 1 + tests/suite/test_rl_policies.py | 1 + tests/suite/test_rl_policies_vsr.py | 1 + tests/suite/test_virtual_server.py | 1 + .../test_virtual_server_canned_responses.py | 1 + .../suite/test_virtual_server_certmanager.py | 3 +++ .../test_virtual_server_external_name.py | 1 + .../suite/test_virtual_server_externaldns.py | 1 + tests/suite/test_virtual_server_grpc.py | 2 ++ tests/suite/test_virtual_server_mixed_grpc.py | 1 + tests/suite/test_virtual_server_redirects.py | 1 + .../suite/test_virtual_server_tls_redirect.py | 1 + tests/suite/test_watch_namespace_label.py | 1 + 23 files changed, 51 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 475d3f0c9c..6d28b92a1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,20 +256,27 @@ jobs: echo "matrix={\"images\": \ [{\"image\": \"debian\", \"marker\": \"ingresses\"}, \ {\"image\": \"alpine\", \"marker\":\"vsr\"}, \ - {\"image\": \"alpine\", \"marker\":\"policies\"}, \ - {\"image\": \"debian\", \"marker\": \"vs\"}, \ + {\"image\": \"alpine\", \"marker\":\"'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'\"}, \ + {\"image\": \"alpine\", \"marker\":\"'policies_rl or policies_ac or policies_jwt or policies_mtls'\"}, \ + {\"image\": \"debian\", \"marker\": \"'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'\"}, \ + {\"image\": \"debian\", \"marker\": \"'vs_ipv6 or vs_rewrite or vs_responses or vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_certmanager'\"}, \ + {\"image\": \"debian\", \"marker\": \"'vs_certmanager'\"}, \ {\"image\": \"ubi\", \"marker\": \"ts\"}, \ - {\"image\": \"debian-plus\", \"marker\": \"vs\"}, \ + {\"image\": \"debian-plus\", \"marker\": \"'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'\"}, \ + {\"image\": \"debian-plus\", \"marker\": \"'vs_ipv6 or vs_rewrite or vs_responses or vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_certmanager'\"}, \ + {\"image\": \"debian-plus\", \"marker\": \"vs_certmanager\"}, \ {\"image\": \"debian-plus\", \"marker\": \"ts\"}, \ {\"image\": \"alpine-plus\", \"marker\":\"ingresses\"}, \ {\"image\": \"alpine-plus\", \"marker\": \"vsr\"}, \ - {\"image\": \"ubi-plus\", \"marker\": \"policies\"}, \ + {\"image\": \"ubi-plus\", \"marker\":\"'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'\"}, \ + {\"image\": \"ubi-plus\", \"marker\":\"'policies_rl or policies_ac or policies_jwt or policies_mtls'\"}, \ {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_integration\"}, \ {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_allow\"}, \ {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_waf_policies and not appprotect_waf_policies_allow'\"}, \ {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_grpc\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch'\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"dos\"}], \ + {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch or appprotect_integration'\"}, \ + {\"image\": \"debian-plus-nap\", \"marker\": \"'dos and not dos_learning'\"}, \ + {\"image\": \"debian-plus-nap\", \"marker\": \"dos_learning\"}], \ \"k8s\": [\"${{ needs.checks.outputs.k8s_latest }}\"]}" >> $GITHUB_OUTPUT else echo "matrix={\"k8s\": [\"1.22.17\", \"1.23.17\", \"1.24.15\", \"1.25.11\", \"1.26.6\", \"1.27.3\", \"${{ needs.checks.outputs.k8s_latest }}\"], \ diff --git a/pyproject.toml b/pyproject.toml index e03093fbb7..2888295493 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,13 +32,26 @@ markers =[ "appprotect_watch", "appprotect_batch", "dos", + "dos_learning", "ingresses", "policies", + "policies_rl", + "policies_jwt", + "policies_ac", + "policies_mtls", "rewrite", "skip_for_nginx_oss", "smoke", "ts", "vs", + "vs_ipv6", + "vs_rewrite", + "vs_responses", + "vs_grpc", + "vs_redirects", + "vs_externaldns", + "vs_externalname", + "vs_certmanager", "vsr", ] testpaths = [ diff --git a/tests/suite/test_ac_policies.py b/tests/suite/test_ac_policies.py index 0fb0771b21..94719ccbb0 100644 --- a/tests/suite/test_ac_policies.py +++ b/tests/suite/test_ac_policies.py @@ -66,6 +66,7 @@ def fin(): @pytest.mark.policies +@pytest.mark.policies_ac @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_ac_policies_vsr.py b/tests/suite/test_ac_policies_vsr.py index 3d052fc4ff..d725021f59 100644 --- a/tests/suite/test_ac_policies_vsr.py +++ b/tests/suite/test_ac_policies_vsr.py @@ -50,6 +50,7 @@ def fin(): @pytest.mark.policies +@pytest.mark.policies_ac @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ diff --git a/tests/suite/test_disable_ipv6.py b/tests/suite/test_disable_ipv6.py index 78d22f1c1f..74aa807b7d 100644 --- a/tests/suite/test_disable_ipv6.py +++ b/tests/suite/test_disable_ipv6.py @@ -21,6 +21,7 @@ @pytest.mark.vs @pytest.mark.ts +@pytest.mark.vs_ipv6 @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup, transport_server_setup", [ diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 016305b042..123418fcec 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -253,6 +253,7 @@ def test_dos_sec_logs_on( assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents assert "bad_actor" in log_contents + @pytest.mark.dos_learning def test_dos_under_attack_with_learning( self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller_with_dos, dos_setup, test_namespace ): diff --git a/tests/suite/test_egress_mtls.py b/tests/suite/test_egress_mtls.py index 1999886b6b..474ae8cb38 100644 --- a/tests/suite/test_egress_mtls.py +++ b/tests/suite/test_egress_mtls.py @@ -50,6 +50,7 @@ def teardown_policy(kube_apis, test_namespace, tls_secret, pol_name, mtls_secret @pytest.mark.policies +@pytest.mark.policies_mtls @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_ingress_mtls.py b/tests/suite/test_ingress_mtls.py index af4390c861..185814ad8e 100644 --- a/tests/suite/test_ingress_mtls.py +++ b/tests/suite/test_ingress_mtls.py @@ -63,6 +63,7 @@ def teardown_policy(kube_apis, test_namespace, tls_secret, pol_name, mtls_secret @pytest.mark.policies +@pytest.mark.policies_mtls @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ @@ -417,6 +418,7 @@ def test_ingress_mtls_policy_cert_crl( @pytest.mark.policies +@pytest.mark.policies_mtls @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ diff --git a/tests/suite/test_jwt_policies.py b/tests/suite/test_jwt_policies.py index bc6b2b11c4..cee9eea96e 100644 --- a/tests/suite/test_jwt_policies.py +++ b/tests/suite/test_jwt_policies.py @@ -27,6 +27,7 @@ @pytest.mark.skip_for_nginx_oss @pytest.mark.policies +@pytest.mark.policies_jwt @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_jwt_policies_vsr.py b/tests/suite/test_jwt_policies_vsr.py index 44c03aaf7a..82e847a898 100644 --- a/tests/suite/test_jwt_policies_vsr.py +++ b/tests/suite/test_jwt_policies_vsr.py @@ -27,6 +27,7 @@ @pytest.mark.skip_for_nginx_oss @pytest.mark.policies +@pytest.mark.policies_jwt @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ diff --git a/tests/suite/test_rewrites.py b/tests/suite/test_rewrites.py index 9ab899ba36..a91d14ce03 100644 --- a/tests/suite/test_rewrites.py +++ b/tests/suite/test_rewrites.py @@ -100,6 +100,7 @@ def fin(): @pytest.mark.parametrize("crd_ingress_controller", [({"type": "complete"})], indirect=True) class TestRewrites: @pytest.mark.vs + @pytest.mark.vs_rewrite @pytest.mark.parametrize("path,args,cookies,expected", test_data) def test_vs_rewrite(self, vs_rewrites_setup, path, args, cookies, expected): """ diff --git a/tests/suite/test_rl_policies.py b/tests/suite/test_rl_policies.py index efed38b4f6..089ffd5734 100644 --- a/tests/suite/test_rl_policies.py +++ b/tests/suite/test_rl_policies.py @@ -25,6 +25,7 @@ @pytest.mark.policies +@pytest.mark.policies_rl @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_rl_policies_vsr.py b/tests/suite/test_rl_policies_vsr.py index c28fc43bf2..b825c4bc87 100644 --- a/tests/suite/test_rl_policies_vsr.py +++ b/tests/suite/test_rl_policies_vsr.py @@ -21,6 +21,7 @@ @pytest.mark.policies +@pytest.mark.policies_rl @pytest.mark.parametrize( "crd_ingress_controller, v_s_route_setup", [ diff --git a/tests/suite/test_virtual_server.py b/tests/suite/test_virtual_server.py index 4e96c764cf..3d13fc131d 100644 --- a/tests/suite/test_virtual_server.py +++ b/tests/suite/test_virtual_server.py @@ -21,6 +21,7 @@ @pytest.mark.vs +@pytest.mark.vs_responses @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", diff --git a/tests/suite/test_virtual_server_canned_responses.py b/tests/suite/test_virtual_server_canned_responses.py index 0248b0fa05..b1da8b38a6 100644 --- a/tests/suite/test_virtual_server_canned_responses.py +++ b/tests/suite/test_virtual_server_canned_responses.py @@ -15,6 +15,7 @@ @pytest.mark.vs +@pytest.mark.vs_responses @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_virtual_server_certmanager.py b/tests/suite/test_virtual_server_certmanager.py index a1c0c6138e..e860317373 100644 --- a/tests/suite/test_virtual_server_certmanager.py +++ b/tests/suite/test_virtual_server_certmanager.py @@ -12,6 +12,7 @@ @pytest.mark.vs +@pytest.mark.vs_certmanager @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller, create_certmanager, virtual_server_setup", @@ -42,6 +43,7 @@ def test_responses_after_setup(self, kube_apis, crd_ingress_controller, create_c @pytest.mark.vs +@pytest.mark.vs_certmanager @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller, create_certmanager, virtual_server_setup", @@ -102,6 +104,7 @@ def test_virtual_server_no_cm(self, kube_apis, crd_ingress_controller, create_ce @pytest.mark.vs +@pytest.mark.vs_certmanager @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller, create_certmanager, virtual_server_setup", diff --git a/tests/suite/test_virtual_server_external_name.py b/tests/suite/test_virtual_server_external_name.py index de5217de7f..89695247cc 100644 --- a/tests/suite/test_virtual_server_external_name.py +++ b/tests/suite/test_virtual_server_external_name.py @@ -85,6 +85,7 @@ def fin(): @pytest.mark.vs +@pytest.mark.vs_externalname @pytest.mark.skip_for_nginx_oss @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", diff --git a/tests/suite/test_virtual_server_externaldns.py b/tests/suite/test_virtual_server_externaldns.py index 86b27e6685..7efa9a7b2b 100644 --- a/tests/suite/test_virtual_server_externaldns.py +++ b/tests/suite/test_virtual_server_externaldns.py @@ -10,6 +10,7 @@ @pytest.mark.vs +@pytest.mark.vs_externaldns @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller_with_ed, create_externaldns, virtual_server_setup", diff --git a/tests/suite/test_virtual_server_grpc.py b/tests/suite/test_virtual_server_grpc.py index 36760127a1..4de034df57 100644 --- a/tests/suite/test_virtual_server_grpc.py +++ b/tests/suite/test_virtual_server_grpc.py @@ -80,6 +80,7 @@ def fin(): @pytest.mark.vs +@pytest.mark.vs_grpc @pytest.mark.smoke @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", @@ -265,6 +266,7 @@ def test_config_after_enable_tls( @pytest.mark.vs +@pytest.mark.vs_gprc @pytest.mark.smoke @pytest.mark.skip_for_nginx_oss @pytest.mark.parametrize( diff --git a/tests/suite/test_virtual_server_mixed_grpc.py b/tests/suite/test_virtual_server_mixed_grpc.py index 267a9fb78f..7abf320f31 100644 --- a/tests/suite/test_virtual_server_mixed_grpc.py +++ b/tests/suite/test_virtual_server_mixed_grpc.py @@ -75,6 +75,7 @@ def fin(): @pytest.mark.vs +@pytest.mark.vs_grpc @pytest.mark.smoke @pytest.mark.flaky(max_runs=3) @pytest.mark.parametrize( diff --git a/tests/suite/test_virtual_server_redirects.py b/tests/suite/test_virtual_server_redirects.py index 7f6066c6b0..1dd03c3e01 100644 --- a/tests/suite/test_virtual_server_redirects.py +++ b/tests/suite/test_virtual_server_redirects.py @@ -13,6 +13,7 @@ @pytest.mark.vs +@pytest.mark.vs_redirects @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_virtual_server_tls_redirect.py b/tests/suite/test_virtual_server_tls_redirect.py index 4310daf488..1e356ac997 100644 --- a/tests/suite/test_virtual_server_tls_redirect.py +++ b/tests/suite/test_virtual_server_tls_redirect.py @@ -7,6 +7,7 @@ @pytest.mark.vs +@pytest.mark.vs_redirects @pytest.mark.parametrize( "crd_ingress_controller, virtual_server_setup", [ diff --git a/tests/suite/test_watch_namespace_label.py b/tests/suite/test_watch_namespace_label.py index 2b80c2aff4..75d3c88d79 100644 --- a/tests/suite/test_watch_namespace_label.py +++ b/tests/suite/test_watch_namespace_label.py @@ -177,6 +177,7 @@ def test_response_codes(self, kube_apis, ingress_controller, backend_setup, expe @pytest.mark.vs +@pytest.mark.vs_responses @pytest.mark.parametrize( "crd_ingress_controller, expected_responses", [