From d62247a6eb138055d070b51c41ec04a97aec2f0b Mon Sep 17 00:00:00 2001 From: tomer pasman Date: Sun, 28 Aug 2022 13:12:04 +0300 Subject: [PATCH 1/5] Clean DoS tests --- tests/suite/dos_utils.py | 23 ++++++++ tests/suite/test_dos.py | 75 ++------------------------ tests/suite/test_virtual_server_dos.py | 5 +- 3 files changed, 32 insertions(+), 71 deletions(-) diff --git a/tests/suite/dos_utils.py b/tests/suite/dos_utils.py index a4406eb156..5d48abba0a 100644 --- a/tests/suite/dos_utils.py +++ b/tests/suite/dos_utils.py @@ -1,6 +1,8 @@ from kubernetes.client import CoreV1Api from kubernetes.stream import stream from suite.resources_utils import get_file_contents, wait_before_test +import os +import subprocess def log_content_to_dic(log_contents): @@ -74,3 +76,24 @@ def get_admd_s_contents(v1: CoreV1Api, pod_name, pod_namespace, time): ) admd_contents = str(resp) return admd_contents + + +def clean_good_bad_clients(): + command = "exec ps -aux | grep good_clients_xff.sh | awk '{print $2}' | xargs kill -9" + + subprocess.Popen( + [command], + preexec_fn=os.setsid, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + command = "exec ps -aux | grep bad_clients_xff.sh | awk '{print $2}' | xargs kill -9" + subprocess.Popen( + [command], + preexec_fn=os.setsid, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 90e2afc5ab..7924df66ae 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -13,7 +13,7 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic +from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic, clean_good_bad_clients from suite.resources_utils import ( clear_file_contents, create_dos_arbitrator, @@ -27,7 +27,6 @@ ensure_response_from_backend, get_file_contents, get_ingress_nginx_template_conf, - get_pods_amount, get_pods_amount_with_name, get_test_file_name, nginx_reload, @@ -77,6 +76,9 @@ def dos_setup( :return: DosSetup """ + # Clean old scripts if still running + clean_good_bad_clients() + print(f"------------- Replace ConfigMap --------------") replace_configmap_from_yaml( kube_apis.v1, @@ -125,6 +127,7 @@ def fin(): delete_common_app(kube_apis, "dos", test_namespace) delete_items_from_yaml(kube_apis, src_sec_yaml, test_namespace) write_to_json(f"reload-{get_test_file_name(request.node.fspath)}.json", reload_times) + clean_good_bad_clients() request.addfinalizer(fin) @@ -233,74 +236,6 @@ def test_dos_sec_logs_on( assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents assert "bad_actor" in log_contents - def test_dos_under_attack_no_learning( - self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller_with_dos, dos_setup, test_namespace - ): - """ - Test App Protect Dos: Block bad clients attack - """ - log_loc = f"/var/log/messages" - print("----------------------- Get syslog pod name ----------------------") - syslog_pod = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "syslog") - assert "syslog" in syslog_pod - clear_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) - - print("------------------------- Deploy ingress -----------------------------") - create_ingress_with_dos_annotations(kube_apis, src_ing_yaml, test_namespace, test_namespace + "/dos-protected") - ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml) - - print("------------------------- Attack -----------------------------") - wait_before_test(10) - print("start bad clients requests") - p_attack = subprocess.Popen( - [f"exec {TEST_DATA}/dos/bad_clients_xff.sh {ingress_host} {dos_setup.req_url}"], - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - print("Attack for 30 seconds") - wait_before_test(30) - - print("Stop Attack") - p_attack.terminate() - - print("wait max 140 seconds after attack stop, to get attack ended") - find_in_log( - kube_apis, - log_loc, - syslog_pod, - ingress_controller_prerequisites.namespace, - 140, - 'attack_event="Attack ended"', - ) - - log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) - log_info_dic = log_content_to_dic(log_contents) - - # Analyze the log - no_attack = False - attack_started = False - under_attack = False - attack_ended = False - for log in log_info_dic: - # Start with no attack - if log["attack_event"] == "No Attack" and int(log["dos_attack_id"]) == 0 and not no_attack: - no_attack = True - # Attack started - elif log["attack_event"] == "Attack started" and int(log["dos_attack_id"]) > 0 and not attack_started: - attack_started = True - # Under attack - elif log["attack_event"] == "Under Attack" and int(log["dos_attack_id"]) > 0 and not under_attack: - under_attack = True - # Attack ended - elif log["attack_event"] == "Attack ended" and int(log["dos_attack_id"]) > 0 and not attack_ended: - attack_ended = True - - delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) - - assert no_attack and attack_started and under_attack and attack_ended - def test_dos_under_attack_with_learning( self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller_with_dos, dos_setup, test_namespace ): diff --git a/tests/suite/test_virtual_server_dos.py b/tests/suite/test_virtual_server_dos.py index 333864885e..1aa9e420b9 100644 --- a/tests/suite/test_virtual_server_dos.py +++ b/tests/suite/test_virtual_server_dos.py @@ -13,7 +13,7 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic +from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic, clean_good_bad_clients from suite.resources_utils import ( clear_file_contents, create_example_app, @@ -100,6 +100,8 @@ def dos_setup( :return: DosSetup """ + # Clean old scripts if still running + clean_good_bad_clients() print(f"------------- Replace ConfigMap --------------") replace_configmap_from_yaml( kube_apis.v1, @@ -133,6 +135,7 @@ def fin(): delete_dos_policy(kube_apis.custom_objects, pol_name, test_namespace) delete_dos_logconf(kube_apis.custom_objects, log_name, test_namespace) delete_dos_protected(kube_apis.custom_objects, protected_name, test_namespace) + clean_good_bad_clients() # delete_items_from_yaml(kube_apis, src_webapp_yaml, test_namespace) # delete_common_app(kube_apis, "dos", test_namespace) # write_to_json(f"reload-{get_test_file_name(request.node.fspath)}.json", reload_times) From 2cad683887f002283622427a2be8a4e0eb034490 Mon Sep 17 00:00:00 2001 From: tomer pasman Date: Tue, 30 Aug 2022 09:09:40 +0300 Subject: [PATCH 2/5] Add admd logs --- tests/suite/test_dos.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 7924df66ae..98badc6226 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -143,6 +143,7 @@ def fin(): f"-enable-custom-resources", f"-enable-app-protect-dos", f"-v=3", + f"-app-protect-dos-debug", ] } ], @@ -189,6 +190,10 @@ def test_ap_nginx_config_entries( delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) + print("\n===================== IC Logs Start =====================") + print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) + print("\n===================== IC Logs End =====================") + for _ in conf_directive: assert _ in result_conf @@ -232,6 +237,10 @@ def test_dos_sec_logs_on( print(log_contents) + print("\n===================== IC Logs Start =====================") + print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) + print("\n===================== IC Logs End =====================") + assert 'product="app-protect-dos"' in log_contents assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents assert "bad_actor" in log_contents @@ -312,6 +321,11 @@ def test_dos_under_attack_with_learning( log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) log_info_dic = log_content_to_dic(log_contents) + print("\n===================== IC Logs Start =====================") + pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") + print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) + print("\n===================== IC Logs End =====================") + # Analyze the log no_attack = False attack_started = False @@ -420,6 +434,11 @@ def test_dos_arbitrator( delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) + print("\n===================== IC Logs Start =====================") + pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") + print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) + print("\n===================== IC Logs End =====================") + assert len(learning_units_hostname) == 2 def test_dos_arbitrator_different_ns( @@ -529,4 +548,9 @@ def test_dos_arbitrator_different_ns( f"{TEST_DATA}/dos/nginx-config.yaml", ) + print("\n===================== IC Logs Start =====================") + pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") + print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) + print("\n===================== IC Logs End =====================") + assert len(learning_units_hostname) == 2 From 135989569ff17d3c3b0954b815805ff6cb4d47ac Mon Sep 17 00:00:00 2001 From: tomer pasman Date: Wed, 31 Aug 2022 12:08:59 +0300 Subject: [PATCH 3/5] Fix lint-python --- tests/suite/dos_utils.py | 5 +++-- tests/suite/test_dos.py | 2 +- tests/suite/test_virtual_server_dos.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/suite/dos_utils.py b/tests/suite/dos_utils.py index 5d48abba0a..0a78790b70 100644 --- a/tests/suite/dos_utils.py +++ b/tests/suite/dos_utils.py @@ -1,8 +1,9 @@ +import os +import subprocess + from kubernetes.client import CoreV1Api from kubernetes.stream import stream from suite.resources_utils import get_file_contents, wait_before_test -import os -import subprocess def log_content_to_dic(log_contents): diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 98badc6226..1d68e326a1 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -13,7 +13,7 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic, clean_good_bad_clients +from suite.dos_utils import check_learning_status_with_admd_s, clean_good_bad_clients, find_in_log, log_content_to_dic from suite.resources_utils import ( clear_file_contents, create_dos_arbitrator, diff --git a/tests/suite/test_virtual_server_dos.py b/tests/suite/test_virtual_server_dos.py index 1aa9e420b9..3a9b93d1c5 100644 --- a/tests/suite/test_virtual_server_dos.py +++ b/tests/suite/test_virtual_server_dos.py @@ -13,7 +13,7 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, find_in_log, log_content_to_dic, clean_good_bad_clients +from suite.dos_utils import check_learning_status_with_admd_s, clean_good_bad_clients, find_in_log, log_content_to_dic from suite.resources_utils import ( clear_file_contents, create_example_app, From ac1d9224c6de0ba7337dd96e1efef015e41cf936 Mon Sep 17 00:00:00 2001 From: tomer pasman Date: Wed, 31 Aug 2022 15:54:44 +0300 Subject: [PATCH 4/5] Remove log prints --- tests/suite/dos_utils.py | 7 ++++++ tests/suite/test_dos.py | 31 ++++++-------------------- tests/suite/test_virtual_server_dos.py | 8 ++++++- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/tests/suite/dos_utils.py b/tests/suite/dos_utils.py index 0a78790b70..83e74b90b0 100644 --- a/tests/suite/dos_utils.py +++ b/tests/suite/dos_utils.py @@ -98,3 +98,10 @@ def clean_good_bad_clients(): stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + + +def print_admd_log(log): + matches = ["ADMD", "DAEMONLESS"] + for line in log.splitlines(): + if any(x in line for x in matches): + print(line) diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py index 1d68e326a1..5d04f8a3f8 100644 --- a/tests/suite/test_dos.py +++ b/tests/suite/test_dos.py @@ -13,7 +13,13 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, clean_good_bad_clients, find_in_log, log_content_to_dic +from suite.dos_utils import ( + check_learning_status_with_admd_s, + clean_good_bad_clients, + find_in_log, + log_content_to_dic, + print_admd_log, +) from suite.resources_utils import ( clear_file_contents, create_dos_arbitrator, @@ -190,10 +196,6 @@ def test_ap_nginx_config_entries( delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) - print("\n===================== IC Logs Start =====================") - print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) - print("\n===================== IC Logs End =====================") - for _ in conf_directive: assert _ in result_conf @@ -237,10 +239,6 @@ def test_dos_sec_logs_on( print(log_contents) - print("\n===================== IC Logs Start =====================") - print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) - print("\n===================== IC Logs End =====================") - assert 'product="app-protect-dos"' in log_contents assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents assert "bad_actor" in log_contents @@ -321,11 +319,6 @@ def test_dos_under_attack_with_learning( log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, ingress_controller_prerequisites.namespace) log_info_dic = log_content_to_dic(log_contents) - print("\n===================== IC Logs Start =====================") - pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") - print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) - print("\n===================== IC Logs End =====================") - # Analyze the log no_attack = False attack_started = False @@ -434,11 +427,6 @@ def test_dos_arbitrator( delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace) - print("\n===================== IC Logs Start =====================") - pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") - print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) - print("\n===================== IC Logs End =====================") - assert len(learning_units_hostname) == 2 def test_dos_arbitrator_different_ns( @@ -548,9 +536,4 @@ def test_dos_arbitrator_different_ns( f"{TEST_DATA}/dos/nginx-config.yaml", ) - print("\n===================== IC Logs Start =====================") - pod_name = self.getPodNameThatContains(kube_apis, ingress_controller_prerequisites.namespace, "nginx-ingress") - print(kube_apis.v1.read_namespaced_pod_log(pod_name, ingress_controller_prerequisites.namespace)) - print("\n===================== IC Logs End =====================") - assert len(learning_units_hostname) == 2 diff --git a/tests/suite/test_virtual_server_dos.py b/tests/suite/test_virtual_server_dos.py index 3a9b93d1c5..8a1cf7a90f 100644 --- a/tests/suite/test_virtual_server_dos.py +++ b/tests/suite/test_virtual_server_dos.py @@ -13,7 +13,13 @@ delete_dos_policy, delete_dos_protected, ) -from suite.dos_utils import check_learning_status_with_admd_s, clean_good_bad_clients, find_in_log, log_content_to_dic +from suite.dos_utils import ( + check_learning_status_with_admd_s, + clean_good_bad_clients, + find_in_log, + log_content_to_dic, + print_admd_log, +) from suite.resources_utils import ( clear_file_contents, create_example_app, From 6e2502fe0acc4d3a877366ce3287dadcbe2951c1 Mon Sep 17 00:00:00 2001 From: tomer pasman Date: Sun, 18 Sep 2022 13:45:06 +0300 Subject: [PATCH 5/5] Remove comments --- tests/suite/test_virtual_server_dos.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/suite/test_virtual_server_dos.py b/tests/suite/test_virtual_server_dos.py index 8a1cf7a90f..5f1a5787cd 100644 --- a/tests/suite/test_virtual_server_dos.py +++ b/tests/suite/test_virtual_server_dos.py @@ -142,9 +142,6 @@ def fin(): delete_dos_logconf(kube_apis.custom_objects, log_name, test_namespace) delete_dos_protected(kube_apis.custom_objects, protected_name, test_namespace) clean_good_bad_clients() - # delete_items_from_yaml(kube_apis, src_webapp_yaml, test_namespace) - # delete_common_app(kube_apis, "dos", test_namespace) - # write_to_json(f"reload-{get_test_file_name(request.node.fspath)}.json", reload_times) request.addfinalizer(fin)