Add pod perf checks for multiple namespaces (#2934)

tests/conftest.py

@@ -12,6 +12,7 @@
     DEFAULT_IMAGE,
     DEFAULT_PULL_POLICY,
     DEFAULT_SERVICE,
+    NS_COUNT,
     NUM_REPLICAS,
 )
 from suite.resources_utils import get_first_pod_name
@@ -94,6 +95,12 @@ def pytest_addoption(parser) -> None:
         default=BATCH_RESOURCES,
         help="Number of VS/Ingress resources to deploy",
     )
+    parser.addoption(
+        "--ns-count",
+        action="store",
+        default=NS_COUNT,
+        help="Number for namespaces to deploy for use in test_multiple_ns_perf.py",
+    )
 
 
 # import fixtures into pytest global namespace
@@ -104,6 +111,7 @@ def pytest_collection_modifyitems(config, items) -> None:
     """
     Skip tests marked with '@pytest.mark.skip_for_nginx_oss' for Nginx OSS runs.
     Skip tests marked with '@pytest.mark.appprotect' for non AP images.
+    Skip tests marked with '@pytest.mark.dos' for non DOS images
 
     :param config: pytest config
     :param items: pytest collected test-items
@@ -140,6 +148,12 @@ def pytest_collection_modifyitems(config, items) -> None:
             if "batch_start" in item.keywords:
                 item.add_marker(batch_start)
 
+    if int(config.getoption("--ns-count")) <= 0:
+        multi_ns = pytest.mark.skip(reason="Skipping watch-namespaces perf. tests")
+        for item in items:
+            if "multi_ns" in item.keywords:
+                item.add_marker(multi_ns)
+
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item) -> None:

tests/settings.py

@@ -15,9 +15,14 @@
 ALLOWED_SERVICE_TYPES = ["nodeport", "loadbalancer"]
 DEFAULT_DEPLOYMENT_TYPE = "deployment"
 ALLOWED_DEPLOYMENT_TYPES = ["deployment", "daemon-set"]
-BATCH_START = "False"
-# Number of Ingress/VS resources to deploy based on BATCH_START value, ref. line #264 in resource_utils.py
-BATCH_RESOURCES = 1
 # Time in seconds to ensure reconfiguration changes in cluster
 RECONFIGURATION_DELAY = 3
 NGINX_API_VERSION = 4
+
+"""Settings below are test specific"""
+# Determines if batch reload tests will be ran or not
+BATCH_START = "False"
+# Number of Ingress/VS resources to deploy based on BATCH_START value, used in test_batch_startup_times.py
+BATCH_RESOURCES = 1
+# Number of namespaces to deploy to measure Pod performance, used in test_multiple_ns_perf.py
+NS_COUNT = 0

tests/suite/custom_resources_utils.py

@@ -408,3 +408,17 @@ def generate_item_with_upstream_options(yaml_manifest, options) -> dict:
     for upstream in dep["spec"]["upstreams"]:
         upstream.update(options)
     return dep
+
+
+def get_pod_metrics(request, namespace) -> list:
+    retries = 0
+    pod_count = int(request.config.getoption("--replicas"))
+    metrics = []
+    while (not metrics) and retries <= (pod_count * 60):
+        metrics = CustomObjectsApi().list_namespaced_custom_object("metrics.k8s.io", "v1beta1", namespace, "pods")[
+            "items"
+        ]
+        time.sleep(1)
+        retries += 1
+        print(f"Retry #{retries}")
+    return metrics

tests/suite/resources_utils.py

@@ -1570,9 +1570,15 @@ def write_to_json(fname, data) -> None:
     file_path = f"{PROJECT_ROOT}/json_files/"
     if not os.path.isdir(file_path):
         os.mkdir(file_path)
-
+    if os.path.exists(f"{file_path}/{fname}"):
+        with open(f"json_files/{fname}") as f:
+            contents = json.load(f)
+        contents.update(data)
+        file = contents
+    else:
+        file = data
     with open(f"json_files/{fname}", "w+") as f:
-        json.dump(data, f, ensure_ascii=False, indent=4)
+        json.dump(file, f, ensure_ascii=False, indent=4)
 
 
 def get_last_log_entry(kube_apis, pod_name, namespace) -> str:

tests/suite/test_multiple_ns_perf.py

@@ -0,0 +1,160 @@
+import time
+from typing import Dict
+
+import pytest
+import yaml
+from settings import TEST_DATA
+from suite.custom_resources_utils import get_pod_metrics
+from suite.resources_utils import (
+    create_example_app,
+    create_ingress,
+    create_ingress_controller,
+    create_namespace_with_name_from_yaml,
+    create_secret_from_yaml,
+    delete_ingress_controller,
+    delete_namespace,
+    get_test_file_name,
+    wait_until_all_pods_are_ready,
+    write_to_json,
+)
+from suite.vs_vsr_resources_utils import create_virtual_server
+
+watched_namespaces = ""
+
+
+def collect_metrics(request, namespace, metric_dict) -> Dict:
+    """Get pod metrics and write them to a json"""
+
+    metrics = get_pod_metrics(request, namespace)
+    metric_dict[f"{request.node.name}+{time.time()}"] = metrics
+    write_to_json(f"pod-metrics-{get_test_file_name(request.node.fspath)}.json", metric_dict)
+
+    return metrics
+
+
+@pytest.fixture(scope="class")
+def ingress_ns_setup(
+    request,
+    kube_apis,
+) -> None:
+    """
+    Create and deploy namespaces, apps and ingresses
+
+    :param request: pytest fixture
+    :param kube_apis: client apis
+    """
+
+    manifest = f"{TEST_DATA}/smoke/standard/smoke-ingress.yaml"
+    ns_count = int(request.config.getoption("--ns-count"))
+    multi_ns = ""
+    for i in range(1, ns_count + 1):
+        watched_namespace = create_namespace_with_name_from_yaml(kube_apis.v1, f"ns-{i}", f"{TEST_DATA}/common/ns.yaml")
+        multi_ns = multi_ns + f"{watched_namespace},"
+        create_example_app(kube_apis, "simple", watched_namespace)
+        secret_name = create_secret_from_yaml(kube_apis.v1, watched_namespace, f"{TEST_DATA}/smoke/smoke-secret.yaml")
+        with open(manifest) as f:
+            doc = yaml.safe_load(f)
+            doc["metadata"]["name"] = f"smoke-ingress-{i}"
+            doc["spec"]["rules"][0]["host"] = f"smoke-{i}.example.com"
+            create_ingress(kube_apis.networking_v1, watched_namespace, doc)
+    global watched_namespaces
+    watched_namespaces = multi_ns[:-1]
+    for i in range(1, ns_count + 1):
+        wait_until_all_pods_are_ready(kube_apis.v1, f"ns-{i}")
+
+    def fin():
+        for i in range(1, ns_count + 1):
+            delete_namespace(kube_apis.v1, f"ns-{i}")
+
+    request.addfinalizer(fin)
+
+
+@pytest.mark.multi_ns
+class TestMultipleSimpleIngress:
+    """Test to output CPU/Memory perf metrics for pods with multiple namespaces (Ingresses)"""
+
+    def test_ingress_multi_ns(
+        self,
+        request,
+        kube_apis,
+        cli_arguments,
+        ingress_ns_setup,
+        ingress_controller_prerequisites,
+    ):
+        metric_dict = {}
+        namespace = ingress_controller_prerequisites.namespace
+        extra_args = ["-enable-custom-resources=false", f"-watch-namespace={watched_namespaces}"]
+        name = create_ingress_controller(kube_apis.v1, kube_apis.apps_v1_api, cli_arguments, namespace, extra_args)
+        metrics = collect_metrics(request, namespace, metric_dict)
+        delete_ingress_controller(kube_apis.apps_v1_api, name, cli_arguments["deployment-type"], namespace)
+
+        assert metrics
+
+
+##############################################################################################################
+
+
+@pytest.fixture(scope="class")
+def vs_ns_setup(
+    request,
+    kube_apis,
+    crds,
+) -> None:
+    """
+    Create and deploy namespaces, apps and ingresses
+
+    :param request: pytest fixture
+    :param kube_apis: client apis
+    :param crds: deploy Custom resource definitions
+    """
+
+    manifest = f"{TEST_DATA}/virtual-server/standard/virtual-server.yaml"
+    ns_count = int(request.config.getoption("--ns-count"))
+    multi_ns = ""
+    for i in range(1, ns_count + 1):
+        watched_namespace = create_namespace_with_name_from_yaml(kube_apis.v1, f"ns-{i}", f"{TEST_DATA}/common/ns.yaml")
+        multi_ns = multi_ns + f"{watched_namespace},"
+        create_example_app(kube_apis, "simple", watched_namespace)
+        with open(manifest) as f:
+            doc = yaml.safe_load(f)
+            doc["metadata"]["name"] = f"virtual-server-{i}"
+            doc["spec"]["host"] = f"virtual-server-{i}.example.com"
+            create_virtual_server(kube_apis.custom_objects, doc, watched_namespace)
+    global watched_namespaces
+    watched_namespaces = multi_ns[:-1]
+    for i in range(1, ns_count + 1):
+        wait_until_all_pods_are_ready(kube_apis.v1, f"ns-{i}")
+
+    def fin():
+        for i in range(1, ns_count + 1):
+            delete_namespace(kube_apis.v1, f"ns-{i}")
+
+    request.addfinalizer(fin)
+
+
+@pytest.mark.multi_ns
+class TestMultipleVS:
+    """Test to output CPU/Memory perf metrics for pods with multiple namespaces (VirtualServers)"""
+
+    def test_vs_multi_ns(
+        self,
+        request,
+        kube_apis,
+        cli_arguments,
+        vs_ns_setup,
+        ingress_controller_prerequisites,
+    ):
+        metric_dict = {}
+        namespace = ingress_controller_prerequisites.namespace
+        extra_args = ["-enable-custom-resources=True", f"-watch-namespace={watched_namespaces}"]
+        name = create_ingress_controller(
+            kube_apis.v1,
+            kube_apis.apps_v1_api,
+            cli_arguments,
+            namespace,
+            extra_args,
+        )
+        metrics = collect_metrics(request, namespace, metric_dict)
+        delete_ingress_controller(kube_apis.apps_v1_api, name, cli_arguments["deployment-type"], namespace)
+
+        assert metrics