Add reload duration outputs to AP tests

deployments/daemon-set/nginx-ingress.yaml

@@ -30,8 +30,8 @@ spec:
           hostPort: 443
         - name: readiness-port
           containerPort: 8081
-       #- name: prometheus
-         #containerPort: 9113
+        - name: prometheus
+          containerPort: 9113
         readinessProbe:
          httpGet:
            path: /nginx-ready

deployments/daemon-set/nginx-plus-ingress.yaml

@@ -30,8 +30,8 @@ spec:
           hostPort: 443
         - name: readiness-port
           containerPort: 8081
-       #- name: prometheus
-         #containerPort: 9113
+        - name: prometheus
+          containerPort: 9113
         readinessProbe:
           httpGet:
             path: /nginx-ready

deployments/deployment/nginx-ingress.yaml

@@ -29,8 +29,8 @@ spec:
           containerPort: 443
         - name: readiness-port
           containerPort: 8081
-       #- name: prometheus
-         #containerPort: 9113
+        - name: prometheus
+          containerPort: 9113
         readinessProbe:
           httpGet:
             path: /nginx-ready

deployments/deployment/nginx-plus-ingress.yaml

@@ -29,8 +29,8 @@ spec:
           containerPort: 443
         - name: readiness-port
           containerPort: 8081
-       #- name: prometheus
-         #containerPort: 9113
+        - name: prometheus
+          containerPort: 9113
         readinessProbe:
           httpGet:
             path: /nginx-ready

tests/.gitignore

@@ -18,3 +18,6 @@ __pycache__/
 
 # Profiling report files
 *.prof
+
+# json artifacts
+json_files/*

tests/suite/fixtures.py

@@ -706,6 +706,10 @@ def __init__(self, public_endpoint: PublicEndpoint, namespace, vs_host, vs_name,
         self.backend_2_url_ssl = (
             f"https://{public_endpoint.public_ip}:{public_endpoint.port_ssl}{vs_paths[1]}"
         )
+        self.metrics_url = (
+            f"http://{public_endpoint.public_ip}:{public_endpoint.metrics_port}/metrics"
+        )
+
 
 
 @pytest.fixture(scope="class")

tests/suite/resources_utils.py

@@ -1,7 +1,9 @@
 """Describe methods to utilize the kubernetes-client."""
-
+import re
+import os
 import time
 import yaml
+import json
 import pytest
 import requests
 
@@ -11,7 +13,7 @@
 from kubernetes import client
 from more_itertools import first
 
-from settings import TEST_DATA, RECONFIGURATION_DELAY, DEPLOYMENTS
+from settings import TEST_DATA, RECONFIGURATION_DELAY, DEPLOYMENTS, PROJECT_ROOT
 
 
 class RBACAuthorization:
@@ -946,6 +948,7 @@ def create_ingress_controller(v1: CoreV1Api, apps_v1_api: AppsV1Api, cli_argumen
     else:
         name = create_daemon_set(apps_v1_api, namespace, dep)
     wait_until_all_pods_are_ready(v1, namespace)
+
     print(f"Ingress Controller was created with name '{name}'")
     return name
 
@@ -1209,7 +1212,7 @@ def ensure_response_from_backend(req_url, host, additional_headers=None, check40
             time.sleep(1)
         pytest.fail(f"Keep getting 502|504 from {req_url} after 60 seconds. Exiting...")
 
-def get_service_endpoint(kube_apis, service_name, namespace):
+def get_service_endpoint(kube_apis, service_name, namespace) -> str:
     """
     Wait for endpoint resource to spin up.
     :param kube_apis: Kubernates API object
@@ -1237,4 +1240,35 @@ def get_service_endpoint(kube_apis, service_name, namespace):
             if ex.status == 500:
                 print("Reason: Internal server error and Request timed out")
                 raise ApiException
-    return ep
+    return ep
+
+def get_last_reload_time(req_url, ingress_class) -> str:
+
+    reload_metric = ""
+    print(req_url)
+    ensure_connection(req_url, 200)
+    resp = requests.get(req_url)
+    assert resp.status_code == 200, f"Expected 200 code for /metrics and got {resp.status_code}"
+    resp_content = resp.content.decode("utf-8")
+    for line in resp_content.splitlines():
+        if 'last_reload_milliseconds{class="%s"}' %ingress_class in line:
+            reload_metric = re.findall("\d+", line)[0]
+            return reload_metric
+
+def get_test_file_name(path) -> str:
+    """
+    :param path: full path to the test file
+    """
+    return (str(path).rsplit('/', 1)[-1])[:-3]
+
+def write_to_json(fname, data) -> None:
+    """
+    :param fname: filename.json
+    :param data: dictionary
+    """
+    file_path = f"{PROJECT_ROOT}/json_files/"
+    if os.path.isdir(file_path) == False:
+        os.mkdir(file_path)
+
+    with open(f"json_files/{fname}", "w+") as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)

tests/suite/test_ac_policies.py

@@ -1,6 +1,12 @@
-import pytest, requests
+import pytest, requests, json
 from kubernetes.client.rest import ApiException
-from suite.resources_utils import wait_before_test, replace_configmap_from_yaml
+from suite.resources_utils import (
+    wait_before_test,
+    replace_configmap_from_yaml,
+    get_last_reload_time,
+    get_test_file_name,
+    write_to_json,
+)
 from suite.custom_resources_utils import (
     read_custom_resource,
     delete_virtual_server,
@@ -33,6 +39,7 @@
 override_vs_spec_route_src = (
     f"{TEST_DATA}/access-control/route-subroute/virtual-server-override-spec-route.yaml"
 )
+reload_times = {}
 
 
 @pytest.fixture(scope="class")
@@ -59,6 +66,10 @@ def fin():
             ingress_controller_prerequisites.namespace,
             std_cm_src,
         )
+        write_to_json(
+            f"reload-{get_test_file_name(request.node.fspath)}.json",
+            reload_times
+        )
 
     request.addfinalizer(fin)
 
@@ -70,9 +81,16 @@ def fin():
         (
             {
                 "type": "complete",
-                "extra_args": [f"-enable-custom-resources", f"-enable-leader-election=false"],
+                "extra_args": [
+                    f"-enable-custom-resources",
+                    f"-enable-leader-election=false",
+                    f"-enable-prometheus-metrics",
+                ],
+            },
+            {
+                "example": "access-control",
+                "app_type": "simple",
             },
-            {"example": "access-control", "app_type": "simple",},
         )
     ],
     indirect=True,
@@ -94,6 +112,7 @@ def restore_default_vs(self, kube_apis, virtual_server_setup) -> None:
     @pytest.mark.smoke
     def test_deny_policy(
         self,
+        request,
         kube_apis,
         crd_ingress_controller,
         virtual_server_setup,
@@ -122,7 +141,9 @@ def test_deny_policy(
         )
         wait_before_test()
 
-        policy_info = read_custom_resource(kube_apis.custom_objects, test_namespace, "policies", pol_name)
+        policy_info = read_custom_resource(
+            kube_apis.custom_objects, test_namespace, "policies", pol_name
+        )
         print(f"\nUse IP listed in deny block: 10.0.0.1")
         resp1 = requests.get(
             virtual_server_setup.backend_1_url,
@@ -136,6 +157,10 @@ def test_deny_policy(
         )
         print(f"Response: {resp2.status_code}\n{resp2.text}")
 
+        reload_ms = get_last_reload_time(virtual_server_setup.metrics_url, "nginx")
+        print(f"last reload duration: {reload_ms} ms")
+        reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
+
         delete_policy(kube_apis.custom_objects, pol_name, test_namespace)
         self.restore_default_vs(kube_apis, virtual_server_setup)
 
@@ -183,7 +208,9 @@ def test_allow_policy(
         )
         wait_before_test()
 
-        policy_info = read_custom_resource(kube_apis.custom_objects, test_namespace, "policies", pol_name)
+        policy_info = read_custom_resource(
+            kube_apis.custom_objects, test_namespace, "policies", pol_name
+        )
         print(f"\nUse IP listed in allow block: 10.0.0.1")
         resp1 = requests.get(
             virtual_server_setup.backend_1_url,
@@ -384,7 +411,12 @@ def test_deleted_policy(
         )
 
     def test_route_override_spec(
-        self, kube_apis, crd_ingress_controller, virtual_server_setup, test_namespace, config_setup,
+        self,
+        kube_apis,
+        crd_ingress_controller,
+        virtual_server_setup,
+        test_namespace,
+        config_setup,
     ):
         """
         Test allow policy specified under routes overrides block in spec

tests/suite/test_app_protect.py

@@ -1,5 +1,5 @@
 import requests
-import pytest, time
+import pytest, json
 
 from settings import TEST_DATA, DEPLOYMENTS
 from suite.custom_resources_utils import (
@@ -19,7 +19,9 @@
     create_ingress_with_ap_annotations,
     ensure_response_from_backend,
     wait_before_test,
-    get_events,
+    get_last_reload_time,
+    get_test_file_name,
+    write_to_json,
 )
 from suite.yaml_utils import get_first_ingress_host_from_yaml
 
@@ -29,6 +31,7 @@
 valid_resp_name = "Server name:"
 invalid_resp_title = "Request Rejected"
 invalid_resp_body = "The requested URL was rejected. Please consult with your administrator."
+reload_times = {}
 
 
 class BackendSetup:
@@ -40,9 +43,10 @@ class BackendSetup:
         ingress_host (str):
     """
 
-    def __init__(self, req_url, req_url_2, ingress_host):
+    def __init__(self, req_url, req_url_2, metrics_url, ingress_host):
         self.req_url = req_url
         self.req_url_2 = req_url_2
+        self.metrics_url = metrics_url
         self.ingress_host = ingress_host
 
 
@@ -62,6 +66,7 @@ def backend_setup(request, kube_apis, ingress_controller_endpoint, test_namespac
     create_example_app(kube_apis, "simple", test_namespace)
     req_url = f"https://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.port_ssl}/backend1"
     req_url_2 = f"https://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.port_ssl}/backend2"
+    metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics"
     wait_until_all_pods_are_ready(kube_apis.v1, test_namespace)
     ensure_connection_to_public_endpoint(
         ingress_controller_endpoint.public_ip,
@@ -99,24 +104,36 @@ def fin():
         delete_common_app(kube_apis, "simple", test_namespace)
         src_sec_yaml = f"{TEST_DATA}/appprotect/appprotect-secret.yaml"
         delete_items_from_yaml(kube_apis, src_sec_yaml, test_namespace)
+        write_to_json(
+            f"reload-{get_test_file_name(request.node.fspath)}.json",
+            reload_times
+        )
 
     request.addfinalizer(fin)
 
-    return BackendSetup(req_url, req_url_2, ingress_host)
+    return BackendSetup(req_url, req_url_2, metrics_url, ingress_host)
 
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
-    [{"extra_args": [f"-enable-custom-resources", f"-enable-app-protect"]}],
+    [
+        {
+            "extra_args": [
+                f"-enable-custom-resources",
+                f"-enable-app-protect",
+                f"-enable-prometheus-metrics",
+            ]
+        }
+    ],
     indirect=True,
 )
 class TestAppProtect:
     @pytest.mark.parametrize("backend_setup", [{"policy": "dataguard-alarm"}], indirect=True)
     def test_responses_dataguard_alarm(
-        self, kube_apis, crd_ingress_controller_with_ap, backend_setup, test_namespace
+        self, request, kube_apis, crd_ingress_controller_with_ap, backend_setup, test_namespace
     ):
         """
         Test dataguard-alarm AppProtect policy: Block malicious script in url
@@ -132,7 +149,12 @@ def test_responses_dataguard_alarm(
         resp_valid = requests.get(
             backend_setup.req_url, headers={"host": backend_setup.ingress_host}, verify=False
         )
+
         print(resp_valid.text)
+        reload_ms = get_last_reload_time(backend_setup.metrics_url, "nginx")
+        print(f"last reload duration: {reload_ms} ms")
+        reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
+
         assert valid_resp_addr in resp_valid.text
         assert valid_resp_name in resp_valid.text
         assert resp_valid.status_code == 200
@@ -150,7 +172,7 @@ def test_responses_dataguard_alarm(
 
     @pytest.mark.parametrize("backend_setup", [{"policy": "file-block"}], indirect=True)
     def test_responses_file_block(
-        self, kube_apis, crd_ingress_controller_with_ap, backend_setup, test_namespace
+        self, request, kube_apis, crd_ingress_controller_with_ap, backend_setup, test_namespace
     ):
         """
         Test file-block AppProtect policy: Block executing types e.g. .bat and .exe
@@ -167,6 +189,11 @@ def test_responses_file_block(
             backend_setup.req_url, headers={"host": backend_setup.ingress_host}, verify=False
         )
         print(resp_valid.text)
+
+        reload_ms = get_last_reload_time(backend_setup.metrics_url, "nginx")
+        print(f"last reload duration: {reload_ms} ms")
+        reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
+
         assert valid_resp_addr in resp_valid.text
         assert valid_resp_name in resp_valid.text
         assert resp_valid.status_code == 200
@@ -280,7 +307,9 @@ def test_responses_user_def_browser(
             backend_setup.req_url, backend_setup.ingress_host, check404=True
         )
 
-        print("----------------------- Send request with User-Agent: browser ----------------------")
+        print(
+            "----------------------- Send request with User-Agent: browser ----------------------"
+        )
 
         headers_firefox = {
             "host": backend_setup.ingress_host,
@@ -326,11 +355,5 @@ def test_responses_user_def_browser(
             and valid_resp_name in resp_safari.text
             and valid_resp_name in resp_custom2.text
         )
-        assert (
-            invalid_resp_title in resp_chrome.text and
-            invalid_resp_title in resp_custom1.text
-        )
-        assert (
-            invalid_resp_body in resp_chrome.text and
-            invalid_resp_body in resp_custom1.text
-        )
+        assert invalid_resp_title in resp_chrome.text and invalid_resp_title in resp_custom1.text
+        assert invalid_resp_body in resp_chrome.text and invalid_resp_body in resp_custom1.text