Allow excluding test groups (#3202)

* Add excluding test groups

* Exclude effective test groups

* Update doc

* Update configuration

* Do not use sets

* Apply suggestions from review

.vscode/launch.json

@@ -22,9 +22,9 @@
             // "args": ["init", "-p", "<PROVIDER>", "-n", "<NAME>", "--full"]
             // "args": ["prepare", "--os", "<OS>", "--arch", "<ARCH>"]
             // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>"]
-            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>", "-g", "<TEST_GROUP>"]
+            // "args": ["--auto-approve", "test", "-b", "${workspaceFolder}/clusters/build/<DIR>", "-i", "<GROUPS>"]
             // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/<DIR>"]
-            // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/<DIR>","--upgrade-components","kafka"]
+            // "args": ["upgrade", "-b", "${workspaceFolder}/clusters/build/<DIR>", "--upgrade-components", "kafka"]
         }
     ]
 }

cli/epicli.py

@@ -21,6 +21,7 @@
 from cli.src.commands.Test import Test
 from cli.src.commands.Upgrade import Upgrade
 from cli.src.Config import Config, SUPPORTED_OS
+from cli.src.helpers.argparse_helpers import comma_separated_type
 from cli.src.helpers.build_io import get_output_path, save_to_file
 from cli.src.helpers.cli_helpers import prompt_for_password, query_yes_no
 from cli.src.helpers.time_helpers import format_time
@@ -277,18 +278,6 @@ def upgrade_parser(subparsers):
         'zookeeper',
         ])
 
-    def comma_separated_type(choices):
-        """Return a function that splits and checks comma-separated values."""
-        def splitarg(arg):
-            values = arg.replace(' ','').lower().split(',')
-            for value in values:
-                if value not in choices:
-                    raise argparse.ArgumentTypeError(
-                        'invalid choice: {!r} (choose from {})'
-                        .format(value, ', '.join(map(repr, choices))))
-            return values
-        return splitarg
-
     #required
     required.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
                             help='Absolute path to directory with build artifacts.')
@@ -332,15 +321,21 @@ def test_parser(subparsers):
                             help='Absolute path to directory with build artifacts.')
 
     #optional
-    group_list = '{' + ', '.join(SpecCommand.get_spec_groups()) + '}'
-    optional.add_argument('-g', '--group', choices=SpecCommand.get_spec_groups(), default='all', action='store', dest='group', required=False, metavar=group_list,
-                            help='Group of tests to be run, e.g. kafka.')
+    TEST_GROUPS = SpecCommand.get_spec_groups()
+    include_choices = ['all'] + TEST_GROUPS
+
+    optional.add_argument('-e', '--exclude', type=comma_separated_type(choices=TEST_GROUPS),
+                            dest='excluded_groups', required=False,
+                            help='Group of tests to be skipped, e.g. -e kafka,kafka_exporter.')
+    optional.add_argument('-i', '--include', default='all', type=comma_separated_type(choices=include_choices),
+                            dest='included_groups', required=False,
+                            help='Group of tests to be run, e.g. -i kafka,kafka_exporter.')
     sub_parser._action_groups.append(optional)
 
     def run_test(args):
         experimental_query()
         adjust_paths_from_build(args)
-        with Test(args) as cmd:
+        with Test(args, TEST_GROUPS) as cmd:
             return cmd.test()
 
     sub_parser.set_defaults(func=run_test)

cli/src/commands/Test.py

@@ -2,16 +2,19 @@
 
 from cli.src.helpers.build_io import (ANSIBLE_INVENTORY_FILE, SPEC_OUTPUT_DIR,
                                       load_manifest)
+from cli.src.helpers.build_io import load_inventory
 from cli.src.helpers.doc_list_helpers import select_single
 from cli.src.spec.SpecCommand import SpecCommand
 from cli.src.Step import Step
 
 
 class Test(Step):
-    def __init__(self, input_data):
+    def __init__(self, input_data, test_groups):
         super().__init__(__name__)
         self.build_directory = input_data.build_directory
-        self.group = input_data.group
+        self.excluded_groups = input_data.excluded_groups
+        self.included_groups = input_data.included_groups
+        self.test_groups = test_groups
 
     def __enter__(self):
         super().__enter__()
@@ -40,8 +43,29 @@ def test(self):
         if not os.path.exists(spec_output):
             os.makedirs(spec_output)
 
+        selected_test_groups = self.included_groups
+
+        # exclude test groups
+        if self.excluded_groups:
+            included_groups = self.included_groups
+            if 'all' in included_groups:
+                # get available test groups
+                inventory_groups = load_inventory(path_to_inventory).list_groups()
+                effective_inventory_groups = inventory_groups + ['common']
+                included_groups = [group for group in self.test_groups if group in effective_inventory_groups]
+
+            selected_test_groups = [group for group in included_groups if group not in self.excluded_groups]
+
         # run the spec tests
-        spec_command = SpecCommand()
-        spec_command.run(spec_output, path_to_inventory, admin_user.name, admin_user.key_path, self.group)
+        if selected_test_groups:
+            spec_command = SpecCommand()
+            if 'all' in selected_test_groups:
+                selected_test_groups = ['all']
+            else:
+                self.logger.info(f'Selected test groups: {", ".join(selected_test_groups)}')
+
+            spec_command.run(spec_output, path_to_inventory, admin_user.name, admin_user.key_path, selected_test_groups)
+        else:
+            raise Exception('No test group specified to run')
 
         return 0

cli/src/helpers/argparse_helpers.py

@@ -0,0 +1,14 @@
+import argparse
+
+
+# Used by multiple epicli parsers
+def comma_separated_type(choices):
+    """Return a function that splits and checks comma-separated values."""
+    def split_arg(arg):
+        values = arg.replace(' ', '').lower().split(',')
+        for value in values:
+            if value not in choices:
+                raise argparse.ArgumentTypeError(
+                    f'invalid choice: {value!r} (choose from {", ".join([repr(choice) for choice in choices])})')
+        return values
+    return split_arg

cli/src/spec/SpecCommand.py

@@ -22,13 +22,13 @@ def check_dependencies(self):
         if  shutil.which('ruby') is None or shutil.which('gem') is None:
             raise Exception(error_str)
 
-        p = subprocess.Popen(['gem', 'query', '--local'], stdout=PIPE)
+        p = subprocess.Popen(['gem', 'list', '--local'], stdout=PIPE)
         out, err = p.communicate()
         if all(n in out.decode('utf-8') for n in required_gems) is False:
             raise Exception(error_str)
 
 
-    def run(self, spec_output, inventory, user, key, group):
+    def run(self, spec_output, inventory, user, key, groups):
         self.check_dependencies()
 
         env = os.environ.copy()
@@ -37,7 +37,7 @@ def run(self, spec_output, inventory, user, key, group):
         env['user'] = user
         env['keypath'] = key
 
-        cmd = f'rake inventory={inventory} user={user} keypath={key} spec_output={spec_output} spec:{group}'
+        cmd = f'rake inventory={inventory} user={user} keypath={key} spec_output={spec_output} spec:{" spec:".join(groups)}'
 
         self.logger.info(f'Running: "{cmd}"')
 
@@ -52,9 +52,11 @@ def run(self, spec_output, inventory, user, key, group):
 
 
     @staticmethod
-    def get_spec_groups():
+    def get_spec_groups() -> list[str]:
+        """Get test groups based on directories."""
+
         listdir = os.listdir(f'{SPEC_TEST_PATH}/spec')
-        groups = ['all']
+        groups = []
         for entry in listdir:
             if os.path.isdir(f'{SPEC_TEST_PATH}/spec/{entry}'):
                 groups = groups + [entry]

docs/home/DEVELOPMENT.md

@@ -173,13 +173,13 @@ The serverspec tests are integrated in Epicli. To run them you can extend the la
         "pythonPath": "${config:python.pythonPath}",
         "env": { "PYTHONPATH": "${workspaceFolder}" },
         "console": "integratedTerminal",
-        "args": ["test", "-b", "${workspaceFolder}/clusters/buildfolder/", "-g", "postgresql"]
+        "args": ["test", "-b", "${workspaceFolder}/clusters/buildfolder/", "-i", "kafka,postgresql"]
     },
 
     ...
   ```
 
-Where the ```-b``` argument points to the build folder of a cluster. The ```-g``` argument can be used to execute a subset of tests and is optional. Omitting ```-g``` will execute all tests.
+Where the ```-b``` argument points to the build folder of a cluster. The ```-i``` argument can be used to execute a subset of tests and is optional. Omitting ```-i``` will execute all tests.
 
 ## Epicli Python dependencies