From 7f2514fe4f24af951cdff24405d524b0319af1c7 Mon Sep 17 00:00:00 2001 From: pasmant <78279234+pasmant@users.noreply.github.com> Date: Thu, 24 Feb 2022 15:21:48 +0200 Subject: [PATCH] Support new directive for dos: app_protect_dos_arb_fqdn --- .../configmap-resource.md | 1 + internal/configs/config_params.go | 1 + internal/configs/configmaps.go | 8 + internal/configs/version1/config.go | 1 + internal/configs/version1/nginx-plus.tmpl | 3 + tests/data/dos/appprotect-dos-arb-svc.yaml | 13 ++ tests/data/dos/appprotect-dos-arb.yaml | 30 ++++ tests/data/dos/arbitrator_ns.yaml | 4 + tests/data/dos/nginx-config-arb-dif-ns.yaml | 13 ++ tests/suite/dos_utils.py | 45 ++++++ tests/suite/ic_fixtures.py | 2 + tests/suite/resources_utils.py | 31 ++-- tests/suite/test_dos.py | 153 ++++++++++++++++-- tests/suite/test_virtual_server_dos.py | 100 +++--------- 14 files changed, 297 insertions(+), 108 deletions(-) create mode 100644 tests/data/dos/appprotect-dos-arb-svc.yaml create mode 100644 tests/data/dos/appprotect-dos-arb.yaml create mode 100644 tests/data/dos/arbitrator_ns.yaml create mode 100644 tests/data/dos/nginx-config-arb-dif-ns.yaml diff --git a/docs/content/configuration/global-configuration/configmap-resource.md b/docs/content/configuration/global-configuration/configmap-resource.md index 4a95a93996..879671bf55 100644 --- a/docs/content/configuration/global-configuration/configmap-resource.md +++ b/docs/content/configuration/global-configuration/configmap-resource.md @@ -188,4 +188,5 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |`app-protect-reconnect-period-seconds` | Sets the `app_protect_reconnect_period_seconds` [global directive](/nginx-app-protect/configuration/#global-directives). | `5` | | |``app-protect-dos-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for Dos Access log traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | `, vs_name_al=$app_protect_dos_vs_name, ip=$remote_addr, tls_fp=$app_protect_dos_tls_fp, outcome=$app_protect_dos_outcome, reason=$app_protect_dos_outcome_reason, policy_name=$app_protect_dos_policy_name, dos_version=$app_protect_dos_version, ip_tls=$remote_addr:$app_protect_dos_tls_fp,` | | |``app-protect-dos-log-format-escaping`` | Sets the characters escaping for the variables of the stream log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | +|``app-protect-dos-arb-fqdn`` | Sets the ``app-protect-dos-arb-fqdn`` [directive](/nginx-app-protect-dos/directives-and-policy/learn-about-directives-and-policy/#arbitrator-fqdn-directive-app_protect_dos_arb_fqdn). | ``svc-appprotect-dos-arb`` | | {{% /table %}} diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index 9683b9c321..07901f3ad6 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -57,6 +57,7 @@ type ConfigParams struct { AppProtectDosResource string MainAppProtectDosLogFormat []string MainAppProtectDosLogFormatEscaping string + MainAppProtectDosArbFqdn string ProxyBuffering bool ProxyBuffers string ProxyBufferSize string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index c60fda9d45..69ab859875 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -524,6 +524,13 @@ func ParseConfigMap(cfgm *v1.ConfigMap, nginxPlus bool, hasAppProtect bool, hasA cfgParams.MainAppProtectDosLogFormatEscaping = appProtectDosLogFormatEscaping } } + + if appProtectDosArbFqdn, exists := cfgm.Data["app-protect-dos-arb-fqdn"]; exists { + appProtectDosArbFqdn = strings.TrimSpace(appProtectDosArbFqdn) + if appProtectDosArbFqdn != "" { + cfgParams.MainAppProtectDosArbFqdn = appProtectDosArbFqdn + } + } } return cfgParams @@ -590,6 +597,7 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config AppProtectReconnectPeriod: config.MainAppProtectReconnectPeriod, AppProtectDosLogFormat: config.MainAppProtectDosLogFormat, AppProtectDosLogFormatEscaping: config.MainAppProtectDosLogFormatEscaping, + AppProtectDosArbFqdn: config.MainAppProtectDosArbFqdn, InternalRouteServer: staticCfgParams.EnableInternalRoutes, InternalRouteServerName: staticCfgParams.InternalRouteServerName, LatencyMetrics: staticCfgParams.EnableLatencyMetrics, diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index d3f9b47e12..7990619bb7 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -217,6 +217,7 @@ type MainConfig struct { AppProtectDosLoadModule bool AppProtectDosLogFormat []string AppProtectDosLogFormatEscaping string + AppProtectDosArbFqdn string InternalRouteServer bool InternalRouteServerName string LatencyMetrics bool diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 3fc0686382..bccd64fc56 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -70,6 +70,9 @@ http { 'outcome=$app_protect_dos_outcome, reason=$app_protect_dos_outcome_reason, ' 'ip_tls=$remote_addr:$app_protect_dos_tls_fp, '; + {{- end}} + {{- if .AppProtectDosArbFqdn}} + app_protect_dos_arb_fqdn {{.AppProtectDosArbFqdn}}; {{- end}} {{- end}} diff --git a/tests/data/dos/appprotect-dos-arb-svc.yaml b/tests/data/dos/appprotect-dos-arb-svc.yaml new file mode 100644 index 0000000000..f86719170f --- /dev/null +++ b/tests/data/dos/appprotect-dos-arb-svc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: svc-appprotect-dos-arb +spec: + selector: + app: appprotect-dos-arb + ports: + - name: arb + port: 3000 + protocol: TCP + targetPort: 3000 + clusterIP: None diff --git a/tests/data/dos/appprotect-dos-arb.yaml b/tests/data/dos/appprotect-dos-arb.yaml new file mode 100644 index 0000000000..31ec0fabb9 --- /dev/null +++ b/tests/data/dos/appprotect-dos-arb.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: appprotect-dos-arb +spec: + replicas: 1 + selector: + matchLabels: + app: appprotect-dos-arb + template: + metadata: + labels: + app: appprotect-dos-arb + spec: + containers: + - name: appprotect-dos-arb + image: docker-registry.nginx.com/nap-dos/app_protect_dos_arb:1.1.0 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: "128Mi" + cpu: "500m" + ports: + - containerPort: 3000 + securityContext: + allowPrivilegeEscalation: false + runAsUser: 1001 + capabilities: + drop: + - ALL diff --git a/tests/data/dos/arbitrator_ns.yaml b/tests/data/dos/arbitrator_ns.yaml new file mode 100644 index 0000000000..d02eb2f820 --- /dev/null +++ b/tests/data/dos/arbitrator_ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: arbitrator diff --git a/tests/data/dos/nginx-config-arb-dif-ns.yaml b/tests/data/dos/nginx-config-arb-dif-ns.yaml new file mode 100644 index 0000000000..2e93ac59a4 --- /dev/null +++ b/tests/data/dos/nginx-config-arb-dif-ns.yaml @@ -0,0 +1,13 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + real-ip-header: "X-Forwarded-For" + real-ip-recursive: "True" + set-real-ip-from: "0.0.0.0/0" + worker-connections: "30000" + worker-rlimit-nofile: "65535" + worker-rlimit-core: "500M" + app-protect-dos-arb-fqdn: "svc-appprotect-dos-arb.arbitrator.svc.cluster.local" diff --git a/tests/suite/dos_utils.py b/tests/suite/dos_utils.py index e35bb5a4fa..a4406eb156 100644 --- a/tests/suite/dos_utils.py +++ b/tests/suite/dos_utils.py @@ -1,3 +1,5 @@ +from kubernetes.client import CoreV1Api +from kubernetes.stream import stream from suite.resources_utils import get_file_contents, wait_before_test @@ -29,3 +31,46 @@ def find_in_log(kube_apis, log_location, syslog_pod, namespace, time, value): retry += 1 wait_before_test(10) print(f"{value} Not in log, retrying... #{retry}") + + +def admd_s_content_to_dic(admd_s_contents): + arr = [] + for line in admd_s_contents.splitlines(): + arr.append(line) + + admd_s_dic = {} + for line in arr: + tmp = line.split(":") + admd_s_dic[tmp[0].split("/")[-1]] = tmp[1] + return admd_s_dic + + +def check_learning_status_with_admd_s(kube_apis, syslog_pod, namespace, time): + retry = 0 + learning_sas = 0.0 + learning_signature = 0.0 + while (learning_sas < 75 or learning_signature != 100) and retry <= time / 15: + retry += 1 + admd_contents = get_admd_s_contents(kube_apis.v1, syslog_pod, namespace, 15) + admd_s_dic = admd_s_content_to_dic(admd_contents) + learn = admd_s_dic["name.info.learning"].replace("[", "").replace("]", "").split(",") + learning_sas = float(learn[0]) + learning_signature = float(learn[3]) + print(f"learning_sas: {learning_sas}, learning_signature: {learning_signature}") + + +def get_admd_s_contents(v1: CoreV1Api, pod_name, pod_namespace, time): + command = ["admd", "-s", "vs."] + resp = stream( + v1.connect_get_namespaced_pod_exec, + pod_name, + pod_namespace, + command=command, + stderr=True, + stdin=False, + stdout=True, + tty=False, + _request_timeout=time, + ) + admd_contents = str(resp) + return admd_contents diff --git a/tests/suite/ic_fixtures.py b/tests/suite/ic_fixtures.py index 4e5de517e0..823c2a0414 100644 --- a/tests/suite/ic_fixtures.py +++ b/tests/suite/ic_fixtures.py @@ -275,6 +275,8 @@ def crd_ingress_controller_with_dos( kube_apis.v1, kube_apis.apps_v1_api, namespace, + f"{DEPLOYMENTS}/deployment/appprotect-dos-arb.yaml", + f"{DEPLOYMENTS}/service/appprotect-dos-arb-svc.yaml", ) print("------------------------- Create IC -----------------------------------") diff --git a/tests/suite/resources_utils.py b/tests/suite/resources_utils.py index ba3333733a..694c6646ad 100644 --- a/tests/suite/resources_utils.py +++ b/tests/suite/resources_utils.py @@ -1126,17 +1126,21 @@ def delete_ingress_controller(apps_v1_api: AppsV1Api, name, dep_type, namespace) delete_daemon_set(apps_v1_api, name, namespace) -def create_dos_arbitrator(v1: CoreV1Api, apps_v1_api: AppsV1Api, namespace) -> str: +def create_dos_arbitrator( + v1: CoreV1Api, apps_v1_api: AppsV1Api, namespace, deployment_yaml_manifest, svc_yaml_manifest +) -> str: """ Create dos arbitrator according to the params. :param v1: CoreV1Api :param apps_v1_api: AppsV1Api :param namespace: namespace name + :param deployment_yaml_manifest: arbitrator deployment yaml file + :param svc_yaml_manifest: arbitrator svc yaml file :return: str """ - yaml_manifest = f"{DEPLOYMENTS}/deployment/appprotect-dos-arb.yaml" - with open(yaml_manifest) as f: + + with open(deployment_yaml_manifest) as f: dep = yaml.safe_load(f) name = create_deployment(apps_v1_api, namespace, dep) @@ -1151,7 +1155,7 @@ def create_dos_arbitrator(v1: CoreV1Api, apps_v1_api: AppsV1Api, namespace) -> s svc_name = create_service_from_yaml( v1, namespace, - f"{DEPLOYMENTS}/service/appprotect-dos-arb-svc.yaml", + svc_yaml_manifest, ) print(f"Dos arbitrator svc was created with name '{svc_name}'") return name @@ -1192,7 +1196,7 @@ def create_ns_and_sa_from_yaml(v1: CoreV1Api, yaml_manifest) -> str: return res["namespace"] -def create_items_from_yaml(kube_apis, yaml_manifest, namespace) -> None: +def create_items_from_yaml(kube_apis, yaml_manifest, namespace) -> {}: """ Apply yaml manifest with multiple items. @@ -1201,22 +1205,27 @@ def create_items_from_yaml(kube_apis, yaml_manifest, namespace) -> None: :param namespace: :return: """ + res = {} print("Load yaml:") with open(yaml_manifest) as f: docs = yaml.safe_load_all(f) for doc in docs: if doc["kind"] == "Secret": - create_secret(kube_apis.v1, namespace, doc) + res["Secret"] = create_secret(kube_apis.v1, namespace, doc) elif doc["kind"] == "ConfigMap": - create_configmap(kube_apis.v1, namespace, doc) + res["ConfigMap"] = create_configmap(kube_apis.v1, namespace, doc) elif doc["kind"] == "Ingress": - create_ingress(kube_apis.networking_v1, namespace, doc) + res["Ingress"] = create_ingress(kube_apis.networking_v1, namespace, doc) elif doc["kind"] == "Service": - create_service(kube_apis.v1, namespace, doc) + res["Service"] = create_service(kube_apis.v1, namespace, doc) elif doc["kind"] == "Deployment": - create_deployment(kube_apis.apps_v1_api, namespace, doc) + res["Deployment"] = create_deployment(kube_apis.apps_v1_api, namespace, doc) elif doc["kind"] == "DaemonSet": - create_daemon_set(kube_apis.apps_v1_api, namespace, doc) + res["DaemonSet"] = create_daemon_set(kube_apis.apps_v1_api, namespace, doc) + elif doc["kind"] == "Namespace": + res["Namespace"] = create_namespace(kube_apis.v1, doc) + + return res def create_ingress_with_ap_annotations( diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 9d8610fcff..90e2afc5ab 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -4,7 +4,7 @@ import pytest import requests -from settings import TEST_DATA +from settings import DEPLOYMENTS, TEST_DATA from suite.custom_resources_utils import ( create_dos_logconf_from_yaml, create_dos_policy_from_yaml, @@ -13,13 +13,15 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import find_in_log, log_content_to_dic +from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic from suite.resources_utils import ( clear_file_contents, + create_dos_arbitrator, create_example_app, create_ingress_with_dos_annotations, create_items_from_yaml, delete_common_app, + delete_dos_arbitrator, delete_items_from_yaml, ensure_connection_to_public_endpoint, ensure_response_from_backend, @@ -325,14 +327,15 @@ def test_dos_under_attack_with_learning( stderr=subprocess.DEVNULL, ) - print("Learning for max 10 minutes") - find_in_log( + print("Learning for max 15 minutes") + nginx_ingress_pod_name = self.getPodNameThatContains( + kube_apis, ingress_controller_prerequisites.namespace, "nginx" + ) + check_learning_status_with_admd_s( kube_apis, - log_loc, - syslog_pod, + nginx_ingress_pod_name, ingress_controller_prerequisites.namespace, - 600, - 'learning_confidence="Ready"', + 900, ) print("------------------------- Attack -----------------------------") @@ -345,8 +348,15 @@ def test_dos_under_attack_with_learning( stderr=subprocess.DEVNULL, ) - print("Attack for 300 seconds") - wait_before_test(300) + print("Wait max 5 Min until finding 3 bad clients") + find_in_log( + kube_apis, + log_loc, + syslog_pod, + ingress_controller_prerequisites.namespace, + 300, + 'bad_actors="3"', + ) print("Stop Attack") p_attack.terminate() @@ -390,8 +400,8 @@ def test_dos_under_attack_with_learning( if not health_ok and float(log["stress_level"]) < 0.6: health_ok = True health_ok_time = datetime.strptime(log["date_time"], fmt) - elif log["attack_event"] == "Attack signature detected": - signature_detected = True + if not signature_detected and int(log["mitigated_by_signatures"]) > 0: + signature_detected = True elif log["attack_event"] == "Bad actors detected": if under_attack: bad_actor_detected = True @@ -412,7 +422,7 @@ def test_dos_under_attack_with_learning( and (health_ok_time - start_attack_time).total_seconds() < 150 and signature_detected and bad_actor_detected - and len(bad_ip) == 0 + and len(bad_ip) <= 1 ) def test_dos_arbitrator( @@ -431,6 +441,95 @@ def test_dos_arbitrator( create_ingress_with_dos_annotations(kube_apis, src_ing_yaml, test_namespace, test_namespace + "/dos-protected") ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml) + print("------------------------- Learning Phase -----------------------------") + print("start good clients requests") + p_good_client = subprocess.Popen( + [f"exec {TEST_DATA}/dos/good_clients_xff.sh {ingress_host} {dos_setup.req_url}"], + preexec_fn=os.setsid, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + print("Learning for max 15 minutes") + find_in_log( + kube_apis, + log_loc, + syslog_pod, + ingress_controller_prerequisites.namespace, + 900, + 'learning_confidence="Ready"', + ) + + print("------------------------- Check new IC pod get info from arbitrator -----------------------------") + ic_ns = ingress_controller_prerequisites.namespace + scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ic_ns, 2) + while get_pods_amount_with_name(kube_apis.v1, "nginx-ingress", "nginx-ingress") != 2: + print(f"Number of replicas is not 2, retrying...") + wait_before_test() + + print("------------------------- Check if new pod receive info from arbitrator -----------------------------") + print("Wait for 60 seconds") + wait_before_test(60) + + log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) + log_info_dic = log_content_to_dic(log_contents) + + print("Stop Good Client") + p_good_client.terminate() + + learning_units_hostname = [] + for log in log_info_dic: + if log["unit_hostname"] not in learning_units_hostname and log["learning_confidence"] == "Ready": + learning_units_hostname.append(log["unit_hostname"]) + + delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) + + assert len(learning_units_hostname) == 2 + + def test_dos_arbitrator_different_ns( + self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller_with_dos, dos_setup, test_namespace + ): + """ + Test App Protect Dos: Check new IC pod get learning info with arbitrator from different namespace + """ + + print("Remove dos arbitrator from namesapce: ", ingress_controller_prerequisites.namespace) + delete_dos_arbitrator( + kube_apis.v1, kube_apis.apps_v1_api, "appprotect-dos-arb", ingress_controller_prerequisites.namespace + ) + + print("------------------------- Create dos arbitrator Namespace -----------------------") + arbitrator_ns_yaml = f"{TEST_DATA}/dos/arbitrator_ns.yaml" + res = create_items_from_yaml(kube_apis, arbitrator_ns_yaml, "") + + print("------------------------- Create dos arbitrator in arbitrator namespace -----------------------") + create_dos_arbitrator( + kube_apis.v1, + kube_apis.apps_v1_api, + res["Namespace"], + f"{TEST_DATA}/dos/appprotect-dos-arb.yaml", + f"{TEST_DATA}/dos/appprotect-dos-arb-svc.yaml", + ) + + print(f"------------- Replace ConfigMap --------------") + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/dos/nginx-config-arb-dif-ns.yaml", + ) + + print("----------------------- Get syslog pod name ----------------------") + syslog_pod = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "syslog") + assert "syslog" in syslog_pod + log_loc = f"/var/log/messages" + clear_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) + + print("------------------------- Deploy ingress -----------------------------") + create_ingress_with_dos_annotations(kube_apis, src_ing_yaml, test_namespace, test_namespace + "/dos-protected") + ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml) + # print("------------------------- Learning Phase -----------------------------") print("start good clients requests") p_good_client = subprocess.Popen( @@ -441,13 +540,13 @@ def test_dos_arbitrator( stderr=subprocess.DEVNULL, ) - print("Learning for max 10 minutes") + print("Learning for max 15 minutes") find_in_log( kube_apis, log_loc, syslog_pod, ingress_controller_prerequisites.namespace, - 600, + 900, 'learning_confidence="Ready"', ) @@ -459,8 +558,8 @@ def test_dos_arbitrator( wait_before_test() print("------------------------- Check if new pod receive info from arbitrator -----------------------------") - print("Wait for 30 seconds") - wait_before_test(30) + print("Wait for 60 seconds") + wait_before_test(60) log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) log_info_dic = log_content_to_dic(log_contents) @@ -475,4 +574,24 @@ def test_dos_arbitrator( delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) + print("Delete namespace: arbitrator") + delete_items_from_yaml(kube_apis, arbitrator_ns_yaml, "") + + print("------------------------- Restore dos arbitrator in nginx namespace -----------------------") + create_dos_arbitrator( + kube_apis.v1, + kube_apis.apps_v1_api, + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/dos/appprotect-dos-arb.yaml", + f"{TEST_DATA}/dos/appprotect-dos-arb-svc.yaml", + ) + + print(f"------------- Restore ConfigMap --------------") + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/dos/nginx-config.yaml", + ) + assert len(learning_units_hostname) == 2 diff --git a/tests/suite/test_virtual_server_dos.py b/tests/suite/test_virtual_server_dos.py index b522621e75..333864885e 100644 --- a/tests/suite/test_virtual_server_dos.py +++ b/tests/suite/test_virtual_server_dos.py @@ -1,6 +1,5 @@ import os import subprocess -import time from datetime import datetime import pytest @@ -14,20 +13,15 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import find_in_log, log_content_to_dic +from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic from suite.resources_utils import ( clear_file_contents, create_example_app, - create_items_from_yaml, delete_common_app, - delete_items_from_yaml, ensure_response_from_backend, get_file_contents, - get_pods_amount, - get_pods_amount_with_name, nginx_reload, replace_configmap_from_yaml, - scale_deployment, wait_before_test, wait_until_all_pods_are_ready, ) @@ -285,7 +279,6 @@ def test_vs_with_dos_config( vs_source = f"{TEST_DATA}/virtual-server-dos/virtual-server.yaml" create_virtual_server_from_yaml(kube_apis.custom_objects, vs_source, test_namespace) - @pytest.mark.skip def test_vs_dos_under_attack_no_learning( self, kube_apis, @@ -382,14 +375,15 @@ def test_dos_under_attack_with_learning( stderr=subprocess.DEVNULL, ) - print("Learning for max 10 minutes") - find_in_log( + print("Learning for max 15 minutes") + nginx_ingress_pod_name = self.getPodNameThatContains( + kube_apis, ingress_controller_prerequisites.namespace, "nginx" + ) + check_learning_status_with_admd_s( kube_apis, - log_loc, - syslog_pod, + nginx_ingress_pod_name, ingress_controller_prerequisites.namespace, - 600, - 'learning_confidence="Ready"', + 900, ) print("------------------------- Attack -----------------------------") @@ -402,8 +396,15 @@ def test_dos_under_attack_with_learning( stderr=subprocess.DEVNULL, ) - print("Attack for 300 seconds") - wait_before_test(300) + print("Wait max 5 Min until finding 3 bad clients") + find_in_log( + kube_apis, + log_loc, + syslog_pod, + ingress_controller_prerequisites.namespace, + 300, + 'bad_actors="3"', + ) print("Stop Attack") p_attack.terminate() @@ -447,8 +448,8 @@ def test_dos_under_attack_with_learning( if not health_ok and float(log["stress_level"]) < 0.6: health_ok = True health_ok_time = datetime.strptime(log["date_time"], fmt) - elif log["attack_event"] == "Attack signature detected": - signature_detected = True + if not signature_detected and int(log["mitigated_by_signatures"]) > 0: + signature_detected = True elif log["attack_event"] == "Bad actors detected": if under_attack: bad_actor_detected = True @@ -467,66 +468,5 @@ def test_dos_under_attack_with_learning( and (health_ok_time - start_attack_time).total_seconds() < 150 and signature_detected and bad_actor_detected - and len(bad_ip) == 0 - ) - - def test_dos_arbitrator( - self, - kube_apis, - ingress_controller_prerequisites, - crd_ingress_controller_with_dos, - virtual_server_setup_dos, - dos_setup, - test_namespace, - ): - """ - Test App Protect Dos: Check new IC pod get learning info - """ - log_loc = f"/var/log/messages" - syslog_pod = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "syslog") - assert "syslog" in syslog_pod - clear_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) - - # print("------------------------- Learning Phase -----------------------------") - print("start good clients requests") - p_good_client = subprocess.Popen( - [f"exec {TEST_DATA}/dos/good_clients_xff.sh {virtual_server_setup_dos.vs_host} {dos_setup.req_url}"], - preexec_fn=os.setsid, - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + and len(bad_ip) <= 1 ) - - print("Learning for max 10 minutes") - find_in_log( - kube_apis, - log_loc, - syslog_pod, - ingress_controller_prerequisites.namespace, - 600, - 'learning_confidence="Ready"', - ) - - print("------------------------- Check new IC pod get info from arbitrator -----------------------------") - ic_ns = ingress_controller_prerequisites.namespace - scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ic_ns, 2) - while get_pods_amount_with_name(kube_apis.v1, "nginx-ingress", "nginx-ingress") != 2: - print(f"Number of replicas is not 2, retrying...") - wait_before_test() - - print("------------------------- Check if new pod receive info from arbitrator -----------------------------") - print("Wait for 30 seconds") - wait_before_test(30) - - log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) - log_info_dic = log_content_to_dic(log_contents) - - print("Stop Good Client") - p_good_client.terminate() - - learning_units_hostname = [] - for log in log_info_dic: - if log["unit_hostname"] not in learning_units_hostname and log["learning_confidence"] == "Ready": - learning_units_hostname.append(log["unit_hostname"]) - - assert len(learning_units_hostname) == 2