Merge pull request #23 from ThorbenCarl/fuzzing/mutational_grey_box

Fuzzing/mutational grey box

.gitignore

@@ -27,4 +27,6 @@ custom_components/loxone/homeassistant
 .vscode
 
 
-.DS_Store
+.DS_Store
+
+.coverage

custom_components/test/fuzzing/fuzzer_overview.puml

@@ -101,6 +101,54 @@ class "MutationalFuzzer" as MFuzzer << class >> {
 '''''''''''''''''''''''''''''''''''''''
 'Runner''''''''''''''''''''''''''''''''
 '''''''''''''''''''''''''''''''''''''''
+'''''''''''''''''''''''''''''''''''''''
+class "GreyBoxFuzzer" as GBFuzzer << class >> {
+    - __RANGE_RANDOM_INT: int = 9
+    - __RANGE_RANDOM_STRING: int = 100
+    - __data_type_creator: DataTypeCreator
+    --
+    - __init__(self) : void
+    + fuzz(self, seed_template: list, seed_specification,: list = None, amount_seeds: int = 100) : list[Seed]
+}
+'''''''''''''''''''''''''''''''''''''''
+class "GreyBoxRunner" as GBRunner << class >> {
+    - __seed_manager = SeedManager()
+    - __mutator = Mutator()
+    --
+    - __init__(self) : void
+    + run(self, function: Callable, seed_population: List[Seed], amount_runs: int = 10000) : list
+    - __hash_md5(self, branch_covered: str) : str
+    - __store_hashed_branch(self, hashed_branch: str) : void
+    - __mutate(self, seed: Seed) : void
+}
+'''''''''''''''''''''''''''''''''''''''
+class "SeedManager" as SeedManager << class >> {
+    - __power_energy: int = 2
+    --
+    - __init__(self) : void
+    + select_seed(self, seed_population: List[Seed]) : Seed
+    + adjust_energy(self, seed: Seed, branch_dict: dict, hashed_branch: str) : void
+    + get_normalized_energy(self, seed_population: List[Seed]) : list
+}
+'''''''''''''''''''''''''''''''''''''''
+class "Seed" as Seed << class >> {
+    - energy: int = 0
+    - seed_values: list = []
+    --
+    - __init__(self, energy: int = 0, seed_values: list = []) : void
+}
+'''''''''''''''''''''''''''''''''''''''
+class "DataTypeCreator" as DataTypeCreator << class >> {
+    - __MAX_INT: int = (1 << 31) - 1
+    - __MAX_UINT = (1 << 32) - 1
+    --
+    - __init__(self) : void
+    - create_int(self, amount_digits: int = 10, random_creation: bool = True) : int
+    - create_float(self, amount_digits: int = 10, random_creation: bool = True) : float
+    - create_string_only_letters(self, amount_chars: int) : int
+    - create_string_special_characters(self, amount_chars: int) : str
+}
+'''''''''''''''''''''''''''''''''''''''
 abstract class "Runner" as runner << abstract class >> {
     --
     - __init__(self) : void
@@ -161,6 +209,29 @@ entity  "test_mut_on_helpers.py" as test_MUT << test cases >>{
     test_get_all() : None
 }
 '''''''''''''''''''''''''''''''''''''''
+entity  "test_grey_box_on_helpers.py" as test_GBox << test case >>{
+    logger
+    grey_box_fuzzer = GreyBoxFuzzer
+    grey_box_runner = GreyBoxRunner
+    --
+
+}
+'''''''''''''''''''''''''''''''''''''''
+entity  "test_grey_box_on_helpers_complex.py" as test_GBox_complex << test case >>{
+    logger
+    grey_box_fuzzer = GreyBoxFuzzer
+    grey_box_runner = GreyBoxRunner
+    --
+}
+'''''''''''''''''''''''''''''''''''''''
+entity  "test_grey_box_on_helpers.py" as test_GBox << test case >>{
+    logger
+    grey_box_fuzzer = GreyBoxFuzzer
+    grey_box_runner = GreyBoxRunner
+    --
+
+}
+'''''''''''''''''''''''''''''''''''''''
 'Fuzzer''''''''''''''''''''''''''''''''
 '''''''''''''''''''''''''''''''''''''''
 fuzzer <|-- VPFuzzer : inherits from <
@@ -169,10 +240,19 @@ GrFuzzer "1"<--* VPFuzzer: has access to <
 gr_pool "1"<-- VPFuzzer: imports <
 fuzzer <|-- GFuzzer : inherits from <
 fuzzer <|-- MFuzzer : inherits from <
+GBFuzzer o-- DataTypeCreator : aggregates >
+GBFuzzer --> Seed: uses >
 '''''''''''''''''''''''''''''''''''''''
 'Runner''''''''''''''''''''''''''''''''
 '''''''''''''''''''''''''''''''''''''''
 runner <|-- PRunner : inherits from <
+GBRunner o-- SeedManager : aggregates >
+GBRunner --> Seed: uses >
+'''''''''''''''''''''''''''''''''''''''
+'Other classes'''''''''''''''''''''''''
+'''''''''''''''''''''''''''''''''''''''
+SeedManager --> Seed: uses >
+DataTypeCreator --> Seed: uses >
 '''''''''''''''''''''''''''''''''''''''
 'Testcases'''''''''''''''''''''''''''''
 '''''''''''''''''''''''''''''''''''''''
@@ -187,4 +267,6 @@ VPFuzzer "1"<-- test_MUT : imports <
 MFuzzer "1"<-- test_MUT : imports <
 GrFuzzer "1"<-- test_MUT : imports <
 PRunner "1"<-- test_MUT : imports <
+GBFuzzer "1"<--* test_GBox : needs a <
+GBRunner "1"<--* test_GBox : needs a <
 @enduml

custom_components/test/fuzzing/fuzzer_utils/GreyBoxFuzzer.py

@@ -0,0 +1,81 @@
+from custom_components.test.fuzzing.fuzzer_utils.Fuzzer import Fuzzer
+from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.Seed import Seed
+from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.DataTypeCreator import DataTypeCreator
+from typing import List
+import random
+
+class GreyBoxFuzzer(Fuzzer):
+    """GreyBox fuzzer class, inherits from the abstract fuzzer class."""
+
+    __RANGE_RANDOM_STRING = 100
+    __data_type_creator = DataTypeCreator()
+
+    def __init__(self):
+        """initialize GreyBoxFuzzer"""
+        print("Initialize GreyBoxFuzzer")
+
+    def fuzz(self, 
+             seed_template: list,
+             seed_specification: list = None,
+             amount_seeds: int = 100) -> List[Seed]:
+        """Returns a population of seeds with specific values based on the seed template and seed specifiction.
+        
+        This function takes two lists 'seed_template' and 'seed_specification' and creates seeds. 
+        The number of seeds is specified by 'amount_seeds'. A list of the random seeds is returned.
+
+        :param seed_template: The seed_template is a list of input types for the function.
+                              The entries must correspond to the valid function parameters of the function to be tested.
+                              Valid inputs are "INT", "FLOAT", "STRING" and "BOOLEAN".
+                              e.g.: ["INT", "FLOAT", "STRING", "BOOLEAN"]
+        :type seed_template: list
+
+        :param seed_specification: A list that provides the number of digits for each data type in seed_template.
+                                   If a random data type is to be initialised anyway, this must be marked with an 'r'.
+                                   Defalault value ist random for every data type.
+                                   E.g.: [5, 2, 'r']
+        :type seed_specification: list
+
+        :param amount_seeds: Amount of seeds which will be created.
+        :type amount_seeds: int
+
+        :return: Returns a list indicating how many tests were successful and how many failed.
+        :rtype: list
+        """
+
+        # Throw exception if seed_specification and seed_template aren't the same length
+        if len(seed_template) != len(seed_specification):
+            raise ValueError("Length of seed_template and seed_specification must be the same length.")
+
+        seed_population = []
+
+        for _ in range(amount_seeds):
+            param_set = []
+            for seed_spec, data_type in zip(seed_specification, seed_template):
+                if data_type == "INT":
+                    if seed_spec == 'r':
+                        param_set.append(self.__data_type_creator.create_int(seed_spec,True))
+                    else:    
+                        param_set.append(self.__data_type_creator.create_int(seed_spec,False))
+
+                elif data_type == "FLOAT":
+                    if seed_spec == 'r':
+                        param_set.append(self.__data_type_creator.create_float(seed_spec,True))
+                    else:
+                        param_set.append(self.__data_type_creator.create_float(seed_spec,False))
+
+                elif data_type == "STRING":
+                    if seed_spec == 'r':
+                        seed_spec = random.randint(1, self.__RANGE_RANDOM_STRING)
+                    rand_val = random.randint(0,1)
+                    if rand_val == 0:
+                        param_set.append(self.__data_type_creator.create_string_only_letters(seed_spec))
+                    elif rand_val == 1: 
+                        param_set.append(self.__data_type_creator.create_string_special_characters(seed_spec))
+
+                elif data_type == "BOOL":
+                    param_set.append(random.choice([True, False]))
+
+            seed = Seed(1, param_set)
+            seed_population.append(seed)
+
+        return seed_population

custom_components/test/fuzzing/fuzzer_utils/GreyBoxRunner.py

@@ -0,0 +1,132 @@
+import logging
+import inspect
+import coverage
+import hashlib
+import random
+from typing import Callable, List
+
+from custom_components.test.fuzzing.fuzzer_utils.Runner import Runner
+from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.Seed import Seed, SeedManager
+from custom_components.test.fuzzing.fuzzer_utils.MutationalFuzzer import MutationalBlackBoxFuzzer
+
+class GreyBoxRunner(Runner):
+    """Greybox runner class, inherits from the abstract runner class."""
+
+    __logger = None
+    __seed_manager = None
+    __mutationalFuzzer = None
+    path_dict = {}
+
+    def __init__(self):
+        """constructor"""
+        self.__logger = logging.getLogger(__name__)
+        self.__seed_manager = SeedManager()
+        self.__mutationalFuzzer = MutationalBlackBoxFuzzer()
+
+    def run(self, function: Callable, seed_population: List[Seed], amount_runs: int = 10000) -> list:
+        """Executes all transferred parameter sets for the transferred function.
+
+        :param function: The passed function which is to be fuzzed.
+        :type function: Callable
+
+        :param seed_population: A list with seeds. A seed is a set of parameters.
+        :type seed_population: list
+
+        :param amount_runs: The number of times the function is to be tested.
+        :param amount_runs: int 
+
+        :return: Returns a dict with 2 keys,
+                 the key 'passed_tests' contains the number of passed tests,
+                 the key 'failed_tests' contains the number of failed tests.
+        :rtype: dict
+        """
+        path_dict = {}
+        path_counter = 0
+
+        sig = inspect.signature(function)
+        num_params = len(sig.parameters)
+        self.__logger.debug(f"The given functions needs {str(num_params)} parameters")
+
+        test_results = {
+            "passed_tests": 0,
+            "failed_tests": 0,
+        }
+
+        for generation in range(0, amount_runs):
+            # get seed for the test
+            seed = self.__seed_manager.select_seed(seed_population)
+
+            # Mutate seed values
+            self.__mutate(seed)
+
+            cov = coverage.Coverage(branch=True)
+            cov.start()
+            try:
+                function(*seed.seed_values)
+                cov.stop()
+                cov.save()
+                test_results["passed_tests"] += 1
+                self.__logger.debug(f"Test {generation} passed with parameters: {seed.seed_values}")
+            except Exception as e:
+                cov.stop()
+                cov.save()
+                test_results["failed_tests"] += 1
+                self.__logger.error(f"Test {generation} failed with parameters: {seed.seed_values}.")
+                self.__logger.error(f"Exception: {e}")
+
+            # check path coverage
+            data = cov.get_data()
+            filename = next(iter(data.measured_files()))
+            path_covered = data.arcs(filename)
+
+            # Create hash of path
+            hashed_path = self.__hash_md5(str(path_covered))
+
+            # Check if a new path was covered
+            if hashed_path not in self.path_dict:
+                self.__logger.debug(f"Newly covered pathes: {path_covered}")
+                seed_population.append(seed)
+                path_counter += 1
+
+
+            # store hash in path_dict
+            path_dict = self.__store_hashed_path(hashed_path, path_dict)
+
+            # Adjust energy of seed
+            self.__seed_manager.adjust_energy(seed, self.path_dict, hashed_path)
+
+        self.__logger.debug("\n#####  Covert pathes  #####\n")
+        self.__logger.debug(f"In total there were {path_counter} pathes discovered")
+
+        return test_results
+
+    def __hash_md5(self, path_covered: str) -> str:
+        md5_hash = hashlib.md5()
+        md5_hash.update(path_covered.encode('utf-8'))
+        return md5_hash.hexdigest()
+
+    def __store_hashed_path(self, hashed_path: str, path_dict: dict) -> dict:
+        if hashed_path in self.path_dict:
+            self.path_dict[hashed_path] += 1
+        else:
+            self.path_dict[hashed_path] = 1
+
+        return path_dict
+
+    def __mutate(self, seed: Seed):
+        """Mutates one of the seed values.
+
+        This function takes a seed and mutates one of the seed values of it.
+
+        :param seed: A seed consists of a list of seed_values.
+        :type seed: Seed
+        """
+        amount_values = len(seed.seed_values)
+        random_index = random.choice(range(0,amount_values))
+        seed_value = seed.seed_values[random_index]
+        seed.seed_values[random_index] = self.__mutationalFuzzer.fuzz([seed_value],2)[1][0]
+
+
+
+
+