Create test_case

custom_components/test/fuzzing/fuzzer_utils/GreyBoxRunner.py

@@ -2,6 +2,7 @@
 import inspect
 import coverage
 import hashlib
+import random
 from typing import Callable, List
 
 from custom_components.test.fuzzing.fuzzer_utils.Runner import Runner
@@ -15,7 +16,7 @@ class GreyBoxRunner(Runner):
     __logger = None
     __seed_manager = None
     __mutator = None
-    branch_dict = {}
+    path_dict = {}
 
     def __init__(self):
         """constructor"""
@@ -40,8 +41,8 @@ def run(self, function: Callable, seed_population: List[Seed], amount_runs: int
                  the key 'failed_tests' contains the number of failed tests.
         :rtype: dict
         """
-        branch_dict = {}
-        branch_counter = 0
+        path_dict = {}
+        path_counter = 0
 
         sig = inspect.signature(function)
         num_params = len(sig.parameters)
@@ -53,16 +54,12 @@ def run(self, function: Callable, seed_population: List[Seed], amount_runs: int
         }
 
         for generation in range(0, amount_runs):
-<<<<<<< HEAD
             # get seed for the test
             seed = self.__seed_manager.select_seed(seed_population)
 
             # Mutate seed values
             self.__mutator.mutate_grey_box_fuzzer(seed)
 
-=======
-            seed = self.__seed_manager.select_seed(seed_population)
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
             cov = coverage.Coverage(branch=True)
             cov.start()
             try:
@@ -78,66 +75,61 @@ def run(self, function: Callable, seed_population: List[Seed], amount_runs: int
                 self.__logger.error(f"Test {generation} failed with parameters: {seed.seed_values}.")
                 self.__logger.error(f"Exception: {e}")
 
-            # check branch coverage
+            # check path coverage
             data = cov.get_data()
             filename = next(iter(data.measured_files()))
-            branch_covered = data.arcs(filename)
+            path_covered = data.arcs(filename)
 
-            # Create hash of branch
-            print(f"Branch: {branch_covered}")
-            hashed_branch = self.__hash_md5(str(branch_covered))
-            print(f"Hashed branch: {hashed_branch}")
+            # Create hash of path
+            ###################print(f"path: {path_covered}")
+            hashed_path = self.__hash_md5(str(path_covered))
+            ###################print(f"Hashed path: {hashed_path}")
 
-            # Check if a new branch was covered
-            if hashed_branch not in self.branch_dict:
-                self.__logger.debug(f"Newly covered branches: {branch_covered}")
-                print(f"Test {generation}, seed_value: {seed.seed_values}, Newly covered branches: {branch_covered}")
-<<<<<<< HEAD
+            # Check if a new path was covered
+            if hashed_path not in self.path_dict:
+                self.__logger.debug(f"Newly covered pathes: {path_covered}")
+                #print(f"Test {generation}, seed_value: {seed.seed_values}, Newly covered pathes: {path_covered}")
                 seed_population.append(seed)
-=======
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
-                branch_counter += 1
-            else:
-                print(f"Test {generation}, seed_value: {seed.seed_values}, No newly covered branches")
-
-            # store hash in branch_dict
-            branch_dict = self.__store_hashed_branch(hashed_branch, branch_dict)
-
+                path_counter += 1
+
 
+            # store hash in path_dict
+            path_dict = self.__store_hashed_path(hashed_path, path_dict)
+
             # Adjust energy of seed
-            print(f"Energy before: {seed.energy}")
-            self.__seed_manager.adjust_energy(seed, self.branch_dict, hashed_branch)
-            print(f"Energy after: {seed.energy}\n")
+            ###################print(f"Energy before: {seed.energy}")
+            self.__seed_manager.adjust_energy(seed, self.path_dict, hashed_path)
+            ###################print(f"Energy after: {seed.energy}\n")
 
-<<<<<<< HEAD
 
-=======
-            # Mutate seed values
-            self.__mutator.mutate_grey_box_fuzzer(seed)
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
 
-        print("\n#####  Hashed branches  #####\n")    
-        print(f"Branch_dict: {self.branch_dict}")
-        print("\n#####  Covert branches  #####\n")
-        print(f"In total there were {branch_counter} branches discovered ")
-<<<<<<< HEAD
+        #print("\n#####  Hashed pathes  #####\n")    
+        #print(f"path_dict: {self.path_dict}")
+        ###################print("\n#####  Covert pathes  #####\n")
+        self.__logger.debug("\n#####  Covert pathes  #####\n")
+        ###################print(f"In total there were {path_counter} pathes discovered")
+        self.__logger.debug(f"In total there were {path_counter} pathes discovered")
+
         print("Population")
         for s in seed_population:
             print(f"{s.seed_values}")
-=======
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
 
         return test_results
 
-    def __hash_md5(self, branch_covered: str) -> str:
+    def __hash_md5(self, path_covered: str) -> str:
         md5_hash = hashlib.md5()
-        md5_hash.update(branch_covered.encode('utf-8'))
+        md5_hash.update(path_covered.encode('utf-8'))
         return md5_hash.hexdigest()
 
-    def __store_hashed_branch(self, hashed_branch: str, branch_dict: dict) -> dict:
-        if hashed_branch in self.branch_dict:
-            self.branch_dict[hashed_branch] += 1
+    def __store_hashed_path(self, hashed_path: str, path_dict: dict) -> dict:
+        if hashed_path in self.path_dict:
+            self.path_dict[hashed_path] += 1
         else:
-            self.branch_dict[hashed_branch] = 1
+            self.path_dict[hashed_path] = 1
 
-        return branch_dict
+        return path_dict
+
+
+
+
+

custom_components/test/fuzzing/fuzzer_utils/MutationalFuzzer.py

@@ -20,7 +20,7 @@ def __init__(self):
             self.__multiplier.append(i)
             i *= 10
 
-    def __delete_random_char(self, string: str) -> str:
+    def delete_random_char(self, string: str) -> str:
         """Returns string with a random character deleted.
 
         This function takes a string `string` as input and returns a new string
@@ -44,7 +44,7 @@ def __delete_random_char(self, string: str) -> str:
         # the substring after the random position.
         return string[:pos] + string[pos + 1 :]
 
-    def __insert_random_char(self, string: str) -> str:
+    def insert_random_char(self, string: str) -> str:
         """Returns string with a random character inserted.
 
         This function takes a string `string` as input and returns a new string
@@ -67,7 +67,7 @@ def __insert_random_char(self, string: str) -> str:
         # and the substring after the random position.
         return string[:pos] + random_character + string[pos:]
 
-    def __flip_random_char(self, string: str) -> str:
+    def flip_random_char(self, string: str) -> str:
         """Returns string with a random bit flipped in a random position.
 
         This function takes a string `string` as input and returns a new string

custom_components/test/fuzzing/fuzzer_utils/fuzzer_tools/Mutator.py

@@ -1,226 +1,62 @@
 from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.Seed import Seed
+from custom_components.test.fuzzing.fuzzer_utils.MutationalFuzzer import MutationalBlackBoxFuzzer
+
 from random import random
 import random
-import math
 
 class Mutator:
+
+    __mutationalFuzzer = None
+
     def __init__(self):
         """initialize Mutator"""
+        self.__mutationalFuzzer = MutationalBlackBoxFuzzer()
 
     def mutate_grey_box_fuzzer(self, seed: Seed):
-        """Mutates all seed values.
+        """Mutates one of the seed values.
 
-        This function takes a seed and mutates all seed values of it.
+        This function takes a seed and mutates one of the seed values of it.
 
         :param seed: A seed consists of a list of seed_values.
         :type seed: Seed
         """
-        for index, seed_value in enumerate(seed.seed_values):
-            if isinstance(seed_value, str):
-                seed.seed_values[index] = self.__grey_mutate_string(seed_value)
-
-
-    def __delete_random_char(self, string: str) -> str:
-        """Returns string with a random character deleted.
-
-        This function takes a string `string` as input and returns a new string
-        with one random character removed from it.
-
-        :param string: Any string from which a character is to be removed.
-        :type string: str
-
-        :return: Returns the input string `string` with one randomly chosen character deleted.
-        :rtype: str
-        """
-        if string == "":
-            # If the string is empty, there's no character to delete, so return the empty string.
-            return string
+        amount_values = len(seed.seed_values)
+        random_index = random.choice(range(0,amount_values))
+        seed_value = seed.seed_values[random_index]
 
-        # Generate a random integer position within the range of the string's indices.
-        pos = random.randint(0, len(string) - 1)
+        if isinstance(seed_value, str):
+            seed.seed_values[random_index] = self.__mutate_string(seed_value)
+        if isinstance(seed_value, int):
+            seed.seed_values[random_index] = self.__mutate_int(seed_value)
 
-        # Create a new string by excluding the character at the random position.
-        # This is done by concatenating the substring before the random position and
-        # the substring after the random position.
-        return string[:pos] + string[pos + 1 :]
-
-    def __insert_random_char(self, string: str) -> str:
-        """Returns string with a random character inserted.
-
-        This function takes a string `string` as input and returns a new string
-        with a random character inserted at a random position within the string.
-
-        :param string: Any string where a character is to be inserted.
-        :type string: str
-
-        :return: Returns the input string `string` with one randomly chosen character inserted at a random position.
-        :rtype: str
-        """
-        # Generate a random position within the range of the string's length (including the end of the string).
-        pos = random.randint(0, len(string))
-
-        # Generate a random character from the ASCII range 32 to 126 (printable characters).
-        random_character = chr(random.randrange(32, 127))
-
-        # Create a new string by inserting the random character at the random position.
-        # This is done by concatenating the substring before the random position, the random character,
-        # and the substring after the random position.
-        return string[:pos] + random_character + string[pos:]
-
-    def __flip_random_char(self, string: str) -> str:
-        """Returns string with a random bit flipped in a random position.
+    def __mutate_string(self, seed_value: str) -> str:
+        """Mutates a string random.
 
-        This function takes a string `string` as input and returns a new string
-        where one randomly chosen character has one of its bits flipped
-        at a random bit position.
+        This function takes a string and applies different mutations on it.
+        1. delete random char
+        2. insert random char
+        3. flip random char
 
-        :param string: Any string where a character's bit is to be flipped.
-        :type string: str
+        :param seed_value: A string which should be mutated.
+        :type seed_value: str
 
-        :return: Returns the input string `string` with one character's bit flipped at a random position.
+        :return: Returns the mutated seed value.
         :rtype: str
         """
-        if string == "":
-            # If the string is empty, there's no character to flip, so return the empty string.
-            return string
-
-        # Generate a random integer position within the range of the string's indices.
-        pos = random.randint(0, len(string) - 1)
-
-        # Get the character at the randomly chosen position.
-        c = string[pos]
-
-        # Generate a random bit position between 0 and 6 (since we are assuming 7-bit ASCII characters).
-        bit = 1 << random.randint(0, 6)
-
-        # Flip the bit at the generated bit position using XOR.
-        new_c = chr(ord(c) ^ bit)
-
-        # Create a new string by replacing the character at the random position with the new character.
-        # This is done by concatenating the substring before the random position, the new character,
-        # and the substring after the random position.
-        return string[:pos] + new_c + string[pos + 1 :]
-
-    def __get_random_float(self) -> float:
-        """Returns a random float value modified by a randomly chosen multiplier.
-
-        This function generates a random float value between 0.0 and 1.0, and then
-        multiplies it by a randomly selected value from the list `self.__multiplier`.
-
-        :return: A random positiv float value.
-        :rtype: float
-        """
-        # Generate a random float between 0.0 and 1.0.
-        random_float = random.random()
-
-        # Multiply the random float by a randomly chosen multiplier from the list `self.__multiplier`.
-        random_float *= random.choice(self.__multiplier)
-
-        # Return the modified random float.
-        return random_float
-
-    def __check_inf(self, number: float) -> float:
-        """Checks if the number is infinite and replaces it with a random value if true.
-
-        This function takes a floating-point number `number` as input. If the number is
-        positive or negative infinity, it replaces the number with a random value between
-        0.0 and 1.0. It also logs this replacement.
-
-        :param number: The number to check for infinity.
-        :type number: float
-
-        :return: Returns the original number if it is not finite; otherwise, returns a random value between 0.0 and 1.0.
-        :rtype: float
-        """
-        if math.isinf(number):
-            # If the number is infinite, replace it with a random value between 0.0 and 1.0.
-            number = random.random()
-            self.__logger.debug(
-                "The return value would be - or + INF, set it to a random value between 0.0 and 1.0"
-            )
-
-        # Return the potentially modified number.
-        return number
-
-    def __add_random_number(self, number: float) -> float:
-        """Returns the input number with a random float added.
-
-        This function takes a floating-point number `number` as input and adds
-        a random float to it. The random float is obtained from the private method
-        `__get_random_float`.
-
-        :param number: The number to which a random float will be added.
-        :type number: float
-
-        :return: Returns the input number `number` with an added random float,
-                 after ensuring the result is not infinite using the `__check_inf` method.
-        :rtype: float
-        """
-        number += self.__get_random_float()
-
-        # Check if the resulting number is infinite.
-        return self.__check_inf(number)
-
-    def __sub_random_number(self, number: float) -> float:
-        """Subtracts a random float from the given number.
-
-        This function takes a float `number` as input and subtracts a randomly
-        generated float from it. The resulting number is then checked for
-        infinity values using the `__check_inf` method.
-
-        :param number: The input number from which a random float will be subtracted.
-        :type number: float
-
-        :return: Returns the resulting number after subtracting a random float and checking for infinity.
-        :rtype: float
-        """
-        number -= self.__get_random_float()
-
-        # Check if the resulting number is infinite.
-        return self.__check_inf(number)
-
-    def __mult_random_number(self, number: float) -> float:
-        """Returns the result of multiplying the input number by a random float.
-
-        This function takes a floating-point number `number` as input and returns
-        a new floating-point number which is the result of multiplying the input
-        number by a randomly generated float. It also checks if the result is
-        infinite.
-
-        :param number: A floating-point number to be multiplied by a random float.
-        :type number: float
-
-        :return: Returns the input number multiplied by a random float,
-                 after checking if the result is infinite.
-        :rtype: float
-        """
-        number *= self.__get_random_float()
-
-        # Check if the resulting number is infinite.
-        return self.__check_inf(number)
-
-    def __div_random_number(self, number: float) -> float:
-        """Divides the input number by a randomly generated float.
-
-        This function takes a float `number` as input and divides it by
-        a random float generated by the `__get_random_float` method. It then
-        returns the result of this division after checking for infinity.
-
-        :param number: The float number to be divided.
-        :type number: float
-
-        :return: Returns the input number divided by a random float.
-        :rtype: float
-        """
-        number /= self.__get_random_float()
-
-        # Check if the resulting number is infinite.
-        return self.__check_inf(number)
+        random_val = random.choice([1, 2, 3])
+        if random_val == 1:
+            seed_value = self.__mutationalFuzzer.delete_random_char(seed_value)
+        elif random_val == 2:
+            seed_value = self.__mutationalFuzzer.insert_random_char(seed_value)
+        elif random_val == 3:
+            seed_value = self.__mutationalFuzzer.flip_random_char(seed_value)
+
+        return seed_value
 
-    def __grey_mutate_string(self, seed_value: str) -> str:
-        """Mutates a string random.
+    def __mutate_int(self, seed_value: str) -> str:
+        """Mutates an integer random.
 
-        This function takes a string and applies different mutations on it.
+        This function takes an integer and applies different mutations on it.
         1. delete random char
         2. insert random char
         3. flip random char
@@ -233,10 +69,10 @@ def __grey_mutate_string(self, seed_value: str) -> str:
         """
         random_val = random.choice([1, 2, 3])
         if random_val == 1:
-            seed_value = self.__delete_random_char(seed_value)
+            seed_value = self.__mutationalFuzzer.delete_random_char(seed_value)
         elif random_val == 2:
-            seed_value = self.__insert_random_char(seed_value)
+            seed_value = self.__mutationalFuzzer.insert_random_char(seed_value)
         elif random_val == 3:
-            seed_value = self.__flip_random_char(seed_value)
+            seed_value = self.__mutationalFuzzer.flip_random_char(seed_value)
 
         return seed_value

custom_components/test/fuzzing/fuzzer_utils/fuzzer_tools/Seed.py

@@ -1,12 +1,6 @@
 from typing import List
-from custom_components.test.fuzzing.fuzzer_utils.ValuePoolFuzzer import ValuePoolFuzzer
-from custom_components.test.fuzzing.fuzzer_utils.ParamRunner import ParamRunner
-from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.DataTypeCreator import DataTypeCreator
 import random
-<<<<<<< HEAD
 import copy
-=======
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
 
 class Seed:
 
@@ -37,18 +31,11 @@ def select_seed(self, seed_population: List[Seed]) -> Seed:
         :return: Returns a single seed.
         :rtype: Seed
         """
-        #print(f"seed population: {seed_population}")
         normalized_energy = self.get_normalized_energy(seed_population)
-        #print(f"normalized energy: {normalized_energy}")
         random_value = random.uniform(0,1)
-        #print(f"random_value: {random_value}")
         for index, normalized_energy_val in enumerate(normalized_energy):
             if random_value <= normalized_energy_val:
-<<<<<<< HEAD
                 seed = copy.deepcopy(seed_population[index])
-=======
-                seed = seed_population[index]
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
                 break
 
         return seed

custom_components/test/fuzzing/grey_box/htest_grey_box_on_helpers.py

@@ -0,0 +1,70 @@
+from custom_components.test.fuzzing.fuzzer_utils.GreyBoxFuzzer import GreyBoxFuzzer
+from custom_components.test.fuzzing.fuzzer_utils.GreyBoxRunner import GreyBoxRunner
+from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.Seed import Seed
+
+from custom_components.loxone.helpers import (
+    map_range,
+    hass_to_lox,
+    lox_to_hass,
+    lox2lox_mapped,
+    lox2hass_mapped,
+    to_hass_color_temp,
+    to_loxone_color_temp,
+    get_room_name_from_room_uuid,
+    get_cat_name_from_cat_uuid,
+    add_room_and_cat_to_value_values,
+    get_miniserver_type,
+    get_all,
+)
+
+# Function to test the grey box fuzzer
+def crashme(s: str) -> None:
+    cnt = 0
+    if len(s) > 0 and s[0] == 'b':
+        cnt += 1
+    if len(s) > 1 and s[1] == 'a':
+        cnt += 1
+    if len(s) > 2 and s[2] == 'd':
+        cnt += 1
+    if len(s) > 3 and s[3] == '!':
+        cnt += 1
+    if cnt >= 3:
+        raise Exception()
+
+
+grey_box_fuzzer = GreyBoxFuzzer()
+grey_box_runner = GreyBoxRunner()
+
+# seed specification
+
+amount_seeds = 10
+seed_template = ["STRING"]
+seed_specification = [4]
+
+
+# create a population with fuzzer
+seed_population = grey_box_fuzzer.fuzz(seed_template, seed_specification, 10)
+
+#seed_1 = Seed(1, ["bear"])
+#seed_2 = Seed(1, ["rats"])
+#seed_3 = Seed(1, ["code"])
+#seed_4 = Seed(1, ["hii!"])
+#seed_5 = Seed(1, ["beer"])
+#seed_6 = Seed(1, ["lol!"])
+#seed_7 = Seed(1, ["bad!"])
+
+#seed_population = [seed_1]#, seed_2, seed_3, seed_4, seed_5, seed_6, seed_7]
+
+# Print seeds in population
+print("#####  Population  #####")
+for i in seed_population:
+    print(i.seed_values)
+
+print("\n#####  Execute Tests  #####\n")
+
+test_results = grey_box_runner.run(crashme, seed_population, 10000)
+
+
+print("\n#####  Test restults  #####\n")
+print(f"Tests passed: {test_results['passed_tests']}")
+print(f"Tests failed: {test_results['failed_tests']}")

custom_components/test/fuzzing/grey_box/test_grey_box_on_helpers.py

@@ -1,6 +1,7 @@
 from custom_components.test.fuzzing.fuzzer_utils.GreyBoxFuzzer import GreyBoxFuzzer
 from custom_components.test.fuzzing.fuzzer_utils.GreyBoxRunner import GreyBoxRunner
-from custom_components.test.fuzzing.fuzzer_utils.fuzzer_tools.Seed import Seed
+import pytest
+import logging
 
 from custom_components.loxone.helpers import (
     map_range,
@@ -10,15 +11,19 @@
     lox2hass_mapped,
     to_hass_color_temp,
     to_loxone_color_temp,
-    get_room_name_from_room_uuid,
-    get_cat_name_from_cat_uuid,
-    add_room_and_cat_to_value_values,
     get_miniserver_type,
-    get_all,
 )
 
+logger = logging.getLogger(__name__)
+grey_box_fuzzer: GreyBoxFuzzer = GreyBoxFuzzer()
+grey_box_runner: GreyBoxRunner = GreyBoxRunner()
+
+
+
+
+
 # Function to test the grey box fuzzer
-def crashme(s: str) -> None:
+def demo_function(s: str) -> None:
     cnt = 0
     if len(s) > 0 and s[0] == 'b':
         cnt += 1
@@ -32,49 +37,14 @@ def crashme(s: str) -> None:
         raise Exception()
 
 
-grey_box_fuzzer = GreyBoxFuzzer()
-grey_box_runner = GreyBoxRunner()
-
-# seed specification
-
-amount_seeds = 10
-seed_template = ["STRING"]
-seed_specification = ['r']
-
-
-# create a population with fuzzer
-#seed_population = grey_box_fuzzer.fuzz(seed_template, seed_specification, 20)
-
-seed_1 = Seed(1, ["bear"])
-<<<<<<< HEAD
-#seed_2 = Seed(1, ["rats"])
-#seed_3 = Seed(1, ["code"])
-#seed_4 = Seed(1, ["hii!"])
-#seed_5 = Seed(1, ["beer"])
-#seed_6 = Seed(1, ["lol!"])
-#seed_7 = Seed(1, ["bad!"])
-
-seed_population = [seed_1]#, seed_2, seed_3, seed_4, seed_5, seed_6, seed_7]
-=======
-seed_2 = Seed(1, ["rats"])
-seed_3 = Seed(1, ["code"])
-seed_4 = Seed(1, ["hii!"])
-seed_5 = Seed(1, ["beer"])
-seed_6 = Seed(1, ["lol!"])
-seed_7 = Seed(1, ["bad!"])
-
-seed_population = [seed_1, seed_2, seed_3, seed_4, seed_5, seed_6, seed_7]
->>>>>>> 29c02c73038c358c0cb8646ae0595b8561485f83
-
-# Print seeds in population
-print("#####  Population  #####")
-for i in seed_population:
-    print(i.seed_values)
+@pytest.mark.skipif(False, reason="Not skiped!")
+def test_crashme() -> None:
+    logger.info("Start of test_crashme() test.")
+    seed_template = ["STRING"]
+    seed_specification = [4]
 
-print("\n#####  Execute Tests  #####\n")
 
-test_results = grey_box_runner.run(crashme, seed_population, 10)
+    seed_population = grey_box_fuzzer.fuzz(seed_template, seed_specification, 1)
+    result = grey_box_runner.run(demo_function, seed_population, 10)
 
-print("\n#####  Test restults  #####\n")
-print(f"Tests passed: {test_results['passed_tests']}")
-print(f"Tests failed: {test_results['failed_tests']}")
+    assert result["failed_tests"] == 0