Add reload and req-resp performance tests for ing and vs (#5048)

perf-tests/README.md

@@ -11,33 +11,41 @@ to find out about various configuration options.
 
 ### Prerequisites
 
-- Minikube.
-- Python3 (in a virtualenv)
+- Any k8s platform of your choice (kind, minikube, GKE, AKS etc.)
+- Python3 and Pytest (in a virtualenv)
 
-#### Step 1 - Create a Minikube Cluster
-
-```bash
-minikube start
-```
+#### Step 1 - Create a cluster on platform of your choice
 
 #### Step 2 - Run the Performance Tests
 
-**Note**: if you have the Ingress Controller deployed in the cluster, please uninstall it first, making sure to remove
+**Note**: if you already have the Ingress Controller deployed in the cluster, please uninstall it first, making sure to remove
 its namespace and RBAC resources.
 
 Run the tests:
 
 - Use local Python3 installation (advised to use pyenv/virtualenv):
 
-    ```bash
+    ```shell
     cd perf_tests
     pip install -r ../tests/requirements.txt --no-deps
-    pytest -v -s -m ap_perf --count=<INT>  --users=<INT> --hatch-rate=<INT> --time=<INT>
     ```
 
-The tests will use the Ingress Controller for NGINX with the image built from `debian-image-nap-plus`. See the section
-below to learn how to configure the tests including the image and the type of NGINX -- NGINX or NGINX Plus. Refer the
-[Configuring the Tests](#configuring-the-tests) section for valid arguments.
+    For Ingress and VS performance tests:
+
+    ```shell
+    pytest -v -s -m perf --count=<INT> --users=<INT> --hatch-rate=<INT> --time=<INT>
+    ```
+
+    For AppProtect performance tests:
+
+    ```shell
+    pytest -v -s -m ap_perf --count=<INT> --users=<INT> --hatch-rate=<INT> --time=<INT>
+    ```
+
+The tests can use the Ingress Controller for NGINX with the image built from `debian-image-nap-plus`, `debian-image-plus`
+or `debian-image`.
+See the section below to learn how to configure the tests including the image and the type of NGINX -- NGINX or
+NGINX Plus. Refer to [Configuring the Tests](#configuring-the-tests) section for valid arguments.
 
 ## Configuring the Tests
 

perf-tests/conftest.py

@@ -132,7 +132,7 @@ def pytest_collection_modifyitems(config, items) -> None:
         for item in items:
             if "skip_for_nginx_plus" in item.keywords:
                 item.add_marker(skip_for_nginx_plus)
-    if "-ap" not in config.getoption("--image"):
+    if "-nap" not in config.getoption("--image"):
         appprotect = pytest.mark.skip(reason="Skip AppProtect test in non-AP image")
         for item in items:
             if "appprotect" in item.keywords:

perf-tests/suite/common.py

@@ -0,0 +1,42 @@
+import re
+import subprocess
+from datetime import datetime
+
+import requests
+
+
+def collect_prom_reload_metrics(metric_list, scenario, ip, port) -> None:
+    req_url = f"http://{ip}:{port}/metrics"
+    resp = requests.get(req_url)
+    resp_decoded = resp.content.decode("utf-8")
+    reload_metric = ""
+    for line in resp_decoded.splitlines():
+        if "last_reload_milliseconds{class" in line:
+            reload_metric = re.findall(r"\d+", line)[0]
+            metric_list.append(
+                {
+                    f"Reload time ({scenario}) ": f"{reload_metric}ms",
+                    "TimeStamp": str(datetime.utcnow()),
+                }
+            )
+
+
+def run_perf(url, setup_users, setup_rate, setup_time, resource):
+    subprocess.run(
+        [
+            "locust",
+            "-f",
+            f"suite/{resource}_request_perf.py",
+            "--headless",
+            "--host",
+            url,
+            "--csv",
+            f"{resource}_response_times",
+            "-u",
+            setup_users,  # total no. of users
+            "-r",
+            setup_rate,  # no. of users hatched per second
+            "-t",
+            setup_time,  # locust session duration in seconds
+        ]
+    )

perf-tests/suite/ing_request_perf.py

@@ -0,0 +1,26 @@
+import os
+
+import yaml
+from locust import HttpUser, task
+
+host = ""
+
+
+class TestResponse(HttpUser):
+    # locust class to be invoked
+    def on_start(self):
+        # get host from appprotect-ingress yaml before each test
+        ing_yaml = os.path.join(os.path.dirname(__file__), "../../tests/data/smoke/standard/smoke-ingress.yaml")
+        with open(ing_yaml) as f:
+            docs = yaml.safe_load_all(f)
+            for dep in docs:
+                self.host = dep["spec"]["rules"][0]["host"]
+        print("Setup finished")
+
+    @task
+    def send_request(self):
+        response = self.client.get(url="", headers={"host": self.host}, verify=False)
+        print(response.text)
+
+    min_wait = 400
+    max_wait = 1400

perf-tests/suite/test_ingress_perf.py

@@ -0,0 +1,113 @@
+import json
+
+import pytest
+import requests
+from common import collect_prom_reload_metrics, run_perf
+from settings import TEST_DATA
+from suite.utils.resources_utils import (
+    create_example_app,
+    create_items_from_yaml,
+    create_secret_from_yaml,
+    delete_common_app,
+    delete_items_from_yaml,
+    delete_secret,
+    ensure_connection,
+    ensure_connection_to_public_endpoint,
+    wait_before_test,
+    wait_until_all_pods_are_ready,
+)
+from suite.utils.yaml_utils import get_first_ingress_host_from_yaml
+
+reload = []
+
+
+class Setup:
+    """
+    Encapsulate the Smoke Example details.
+
+    Attributes:
+        public_endpoint (PublicEndpoint):
+    """
+
+    def __init__(self, req_url):
+        self.req_url = req_url
+
+
+@pytest.fixture(scope="class")
+def setup(request, kube_apis, ingress_controller_endpoint, test_namespace) -> Setup:
+    print("------------------------- Deploy prerequisites -----------------------------------")
+    secret_name = create_secret_from_yaml(kube_apis.v1, test_namespace, f"{TEST_DATA}/smoke/smoke-secret.yaml")
+
+    create_example_app(kube_apis, "simple", test_namespace)
+    wait_until_all_pods_are_ready(kube_apis.v1, test_namespace)
+    req_url = f"https://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.port_ssl}/backend1"
+    ensure_connection_to_public_endpoint(
+        ingress_controller_endpoint.public_ip,
+        ingress_controller_endpoint.port,
+        ingress_controller_endpoint.port_ssl,
+    )
+
+    def fin():
+        print("Clean up simple app")
+        delete_common_app(kube_apis, "simple", test_namespace)
+        delete_secret(kube_apis.v1, secret_name, test_namespace)
+        with open("reload_ing.json", "w+") as f:
+            json.dump(reload, f, ensure_ascii=False, indent=4)
+
+    request.addfinalizer(fin)
+    return Setup(req_url)
+
+
+@pytest.fixture
+def setup_users(request):
+    return request.config.getoption("--users")
+
+
+@pytest.fixture
+def setup_rate(request):
+    return request.config.getoption("--hatch-rate")
+
+
+@pytest.fixture
+def setup_time(request):
+    return request.config.getoption("--time")
+
+
+@pytest.mark.perf
+@pytest.mark.parametrize(
+    "ingress_controller",
+    [
+        {
+            "extra_args": [
+                f"-enable-prometheus-metrics",
+            ]
+        }
+    ],
+    indirect=["ingress_controller"],
+)
+class TestIngressPerf:
+    def test_perf(
+        self,
+        kube_apis,
+        ingress_controller_endpoint,
+        test_namespace,
+        ingress_controller,
+        setup,
+        setup_users,
+        setup_rate,
+        setup_time,
+    ):
+        create_items_from_yaml(kube_apis, f"{TEST_DATA}/smoke/standard/smoke-ingress.yaml", test_namespace)
+        ingress_host = get_first_ingress_host_from_yaml(f"{TEST_DATA}/smoke/standard/smoke-ingress.yaml")
+        wait_before_test()
+        ensure_connection(setup.req_url, 200, {"host": ingress_host})
+        resp = requests.get(setup.req_url, headers={"host": ingress_host}, verify=False)
+        assert resp.status_code == 200
+        collect_prom_reload_metrics(
+            reload,
+            "Ingress resource",
+            ingress_controller_endpoint.public_ip,
+            ingress_controller_endpoint.metrics_port,
+        )
+        run_perf(setup.req_url, setup_users, setup_rate, setup_time, "ing")
+        delete_items_from_yaml(kube_apis, f"{TEST_DATA}/smoke/standard/smoke-ingress.yaml", test_namespace)

perf-tests/suite/test_vs_perf.py

@@ -0,0 +1,80 @@
+import json
+import re
+import subprocess
+from datetime import datetime
+
+import pytest
+import requests
+from common import collect_prom_reload_metrics, run_perf
+from suite.utils.resources_utils import wait_before_test
+
+reload = []
+
+
+@pytest.fixture(scope="class")
+def collect(request, kube_apis, ingress_controller_endpoint, test_namespace) -> None:
+    def fin():
+        with open("reload_vs.json", "w+") as f:
+            json.dump(reload, f, ensure_ascii=False, indent=4)
+
+    request.addfinalizer(fin)
+
+
+@pytest.fixture
+def setup_users(request):
+    return request.config.getoption("--users")
+
+
+@pytest.fixture
+def setup_rate(request):
+    return request.config.getoption("--hatch-rate")
+
+
+@pytest.fixture
+def setup_time(request):
+    return request.config.getoption("--time")
+
+
+@pytest.mark.perf
+@pytest.mark.parametrize(
+    "crd_ingress_controller, virtual_server_setup",
+    [
+        (
+            {
+                "type": "complete",
+                "extra_args": [f"-enable-custom-resources", f"-enable-prometheus-metrics"],
+            },
+            {
+                "example": "virtual-server",
+                "app_type": "simple",
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestVirtualServerPerf:
+    def test_vs_perf(
+        self,
+        kube_apis,
+        ingress_controller_endpoint,
+        crd_ingress_controller,
+        virtual_server_setup,
+        collect,
+        setup_rate,
+        setup_time,
+        setup_users,
+    ):
+        wait_before_test()
+        resp = requests.get(
+            virtual_server_setup.backend_1_url,
+            headers={"host": virtual_server_setup.vs_host},
+        )
+        assert resp.status_code == 200
+        collect_prom_reload_metrics(
+            reload,
+            "VS resource",
+            ingress_controller_endpoint.public_ip,
+            ingress_controller_endpoint.metrics_port,
+        )
+
+        run_perf(virtual_server_setup.backend_1_url, setup_users, setup_rate, setup_time, "vs")

perf-tests/suite/vs_request_perf.py

@@ -0,0 +1,28 @@
+import os
+
+import yaml
+from locust import HttpUser, task
+
+host = ""
+
+
+class TestResponse(HttpUser):
+    # locust class to be invoked
+    def on_start(self):
+        # get host from appprotect-ingress yaml before each test
+        ing_yaml = os.path.join(
+            os.path.dirname(__file__), "../../tests/data/virtual-server/standard/virtual-server.yaml"
+        )
+        with open(ing_yaml) as f:
+            docs = yaml.safe_load_all(f)
+            for dep in docs:
+                self.host = dep["spec"]["host"]
+        print("Setup finished")
+
+    @task
+    def send_request(self):
+        response = self.client.get(url="", headers={"host": self.host}, verify=False)
+        print(response.text)
+
+    min_wait = 400
+    max_wait = 1400