From e4a8be028e4accaa1c2dc140ddc08628e58cd7e2 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 6 Feb 2024 19:52:21 -0300 Subject: [PATCH 01/23] Implement logger to workflow_engine and improve log messages --- .../workflow_engine/logging/__init__.py | 0 .../workflow_engine/logging/config.yaml | 28 ++++++++ .../modules/workflow_engine/logging/filter.py | 20 ++++++ .../modules/workflow_engine/logging/logger.py | 69 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 deployability/modules/workflow_engine/logging/__init__.py create mode 100644 deployability/modules/workflow_engine/logging/config.yaml create mode 100644 deployability/modules/workflow_engine/logging/filter.py create mode 100644 deployability/modules/workflow_engine/logging/logger.py diff --git a/deployability/modules/workflow_engine/logging/__init__.py b/deployability/modules/workflow_engine/logging/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deployability/modules/workflow_engine/logging/config.yaml b/deployability/modules/workflow_engine/logging/config.yaml new file mode 100644 index 0000000000..8f9a6884d0 --- /dev/null +++ b/deployability/modules/workflow_engine/logging/config.yaml @@ -0,0 +1,28 @@ +version: 1 +formatters: + simple: + format: '[%(asctime)s] [%(levelname)s] [%(process)d] [%(threadName)s] [%(name)s]: %(message)s' + colored: + (): colorlog.ColoredFormatter + format: '%(log_color)s[%(asctime)s] [%(levelname)s] [%(process)d] [%(threadName)s] [%(name)s]: %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + log_colors: + DEBUG: cyan + INFO: green + WARNING: yellow + ERROR: red + CRITICAL: red,bg_white +handlers: + console: + class: colorlog.StreamHandler + level: DEBUG + formatter: colored + stream: ext://sys.stdout + file: + class: logging.FileHandler + level: DEBUG + formatter: simple + filename: tmp/flowbacca/logs.log +root: + level: DEBUG + handlers: [console, file] diff --git a/deployability/modules/workflow_engine/logging/filter.py b/deployability/modules/workflow_engine/logging/filter.py new file mode 100644 index 0000000000..9ef98fa179 --- /dev/null +++ b/deployability/modules/workflow_engine/logging/filter.py @@ -0,0 +1,20 @@ +import logging +import threading + + +class ThreadIDFilter(logging.Filter): + """ + A filter that uppercases the name of the log record. + """ + def filter(self, record: str) -> bool: + """ + Inject thread_id to log records. + + Args: + record (LogRecord): The log record to filter. + + Returns: + bool: True if the record should be logged, False otherwise. + """ + record.thread_id = threading.get_native_id() + return record diff --git a/deployability/modules/workflow_engine/logging/logger.py b/deployability/modules/workflow_engine/logging/logger.py new file mode 100644 index 0000000000..2cae892a1b --- /dev/null +++ b/deployability/modules/workflow_engine/logging/logger.py @@ -0,0 +1,69 @@ +import logging +import logging.config +from pathlib import Path +import threading + +import yaml + +# The logging module will use the generic logger anyway +# that is because the logging module is a singleton. +# So, we can just use it this way here, and it will work. +# formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(process)d] [%(threadName)s] %(name)s: %(message)s') + +def _load_config() -> None: + """ + Loads the logging configuration from 'config.yaml' file. + """ + config_path = Path(__file__).parent / 'config.yaml' + with open(config_path, 'r') as f: + config = yaml.safe_load(f.read()) + logging.config.dictConfig(config) + +_load_config() + + # self.logger = logging.getLogger(name) +logger = logging.getLogger("workflow_engine") + +# def parse_multiline_log(logger, log_message): +# """ +# Parses a multiline log message and logs it with the provided logger. + +# Args: +# logger (logging.Logger): The logger instance. +# log_message (str): The log message to parse and log. +# """ +# for line in log_message.split('\n'): +# if line: +# parse_and_log(logger, line) + +# # log_utils.py +# import logging +# import os +# import threading + +# def setup_logger(name): +# logger = logging.getLogger(name) +# handler = logging.StreamHandler() +# formatter = logging.Formatter('%(asctime)s [PID: %(process)d] [%(threadName)s] [%(levelname)s] %(message)s') +# handler.setFormatter(formatter) +# logger.addHandler(handler) +# logger.setLevel(logging.DEBUG) +# return logger + +# def parse_and_log(logger: logging.Logger, line: str, extra: dict = None): +# # Parse the log message +# try: +# _, log_level, log_message = line.split('[', 2) +# log_message = log_message.rstrip(']') +# except ValueError: +# log_message = line +# log_level = 'DEBUG' + +# if log_level.strip().upper() == 'DEBUG': +# logger.debug(log_message, extra=extra) +# elif log_level.strip().upper() == 'INFO': +# logger.info(log_message, extra=extra) +# elif log_level.strip().upper() == 'WARNING': +# logger.warning(log_message, extra=extra) +# elif log_level.strip().upper() == 'ERROR': +# logger.error(log_message, extra=extra) From 43206e5b05fe07c2d8653324f5a55f67214948e3 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 6 Feb 2024 20:25:09 -0300 Subject: [PATCH 02/23] Apply logger to the workflow_engine files --- deployability/launchers/workflow_engine.py | 5 +--- .../modules/workflow_engine/__init__.py | 1 - .../workflow_engine/logging/config.yaml | 2 +- .../workflow_engine/schema_validator.py | 4 +-- deployability/modules/workflow_engine/task.py | 11 +++----- .../modules/workflow_engine/utils.py | 6 ----- .../workflow_engine/workflow_processor.py | 27 ++++++++++--------- 7 files changed, 21 insertions(+), 35 deletions(-) delete mode 100644 deployability/modules/workflow_engine/utils.py diff --git a/deployability/launchers/workflow_engine.py b/deployability/launchers/workflow_engine.py index 3aba267f5c..dc804bb34c 100755 --- a/deployability/launchers/workflow_engine.py +++ b/deployability/launchers/workflow_engine.py @@ -5,8 +5,6 @@ import os import sys import argparse -import logging -import colorlog project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) sys.path.append(project_root) @@ -17,8 +15,7 @@ def parse_arguments() -> argparse.Namespace: """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description='Execute tasks in a workflow.') + parser = argparse.ArgumentParser(description='Execute tasks in a workflow.') parser.add_argument('workflow_file', type=str,help='Path to the workflow file (YAML format).') parser.add_argument('--threads', type=int, default=1, required=False, help='Number of threads to use for parallel execution.') parser.add_argument('--dry-run', action='store_true', required=False, help='Display the plan without executing tasks.') diff --git a/deployability/modules/workflow_engine/__init__.py b/deployability/modules/workflow_engine/__init__.py index 5627726b79..338f58e9fd 100755 --- a/deployability/modules/workflow_engine/__init__.py +++ b/deployability/modules/workflow_engine/__init__.py @@ -1,2 +1 @@ from .workflow_processor import WorkflowProcessor - diff --git a/deployability/modules/workflow_engine/logging/config.yaml b/deployability/modules/workflow_engine/logging/config.yaml index 8f9a6884d0..764f4cc7a3 100644 --- a/deployability/modules/workflow_engine/logging/config.yaml +++ b/deployability/modules/workflow_engine/logging/config.yaml @@ -22,7 +22,7 @@ handlers: class: logging.FileHandler level: DEBUG formatter: simple - filename: tmp/flowbacca/logs.log + filename: /tmp/flowbacca.log root: level: DEBUG handlers: [console, file] diff --git a/deployability/modules/workflow_engine/schema_validator.py b/deployability/modules/workflow_engine/schema_validator.py index 6e69a6db95..99b1db77a9 100755 --- a/deployability/modules/workflow_engine/schema_validator.py +++ b/deployability/modules/workflow_engine/schema_validator.py @@ -5,7 +5,7 @@ from pathlib import Path from ruamel.yaml import YAML -from modules.generic.logger import Logger +from .logging.logger import logger class SchemaValidator: @@ -28,7 +28,7 @@ def __init__(self, schema: Path | str, to_validate: Path | str): schema_data: str = None yaml_data: str = None - self.logger = Logger(Path(__file__).stem).get_logger() + self.logger = logger with open(schema, 'r') as schema_file: self.logger.debug(f"Loading schema file: {schema}") schema_data = json.load(schema_file) diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index 8903119142..e9695feed6 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -7,7 +7,7 @@ import time from abc import ABC, abstractmethod -from .utils import logger +from .logging.logger import logger class Task(ABC): @@ -45,12 +45,10 @@ def execute(self) -> None: elif isinstance(arg, dict): key, value = list(arg.items())[0] if isinstance(value, list): - for argvalue in value: - print(f"argvalue {argvalue}") task_args.extend([f"--{key}={argvalue}" for argvalue in value]) else: task_args.append(f"--{key}={value}") - print(f"task_args {task_args}") + self.logger.debug(f'Running task "{self.task_name}" with arguments: {task_args}') result = None try: result = subprocess.run( @@ -59,10 +57,7 @@ def execute(self) -> None: capture_output=True, text=True, ) - - logger.info(str(result.stdout)) - logger.info("%s: %s", "Finish task: ", self.task_name, extra={'tag': self.task_name}) - + logger.debug(f'Finished task "{self.task_name}" execution with result:\n{str(result.stdout)}') if result.returncode != 0: raise subprocess.CalledProcessError(returncode=result.returncode, cmd=result.args, output=result.stdout) diff --git a/deployability/modules/workflow_engine/utils.py b/deployability/modules/workflow_engine/utils.py deleted file mode 100644 index 3d64f331a8..0000000000 --- a/deployability/modules/workflow_engine/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -import logging - -# The logging module will use the generic logger anyway -# that is because the logging module is a singleton. -# So, we can just use it this way here, and it will work. -logger = (lambda: logging.getLogger("workflow_engine"))() diff --git a/deployability/modules/workflow_engine/workflow_processor.py b/deployability/modules/workflow_engine/workflow_processor.py index 0a22727a06..f5e2be859a 100755 --- a/deployability/modules/workflow_engine/workflow_processor.py +++ b/deployability/modules/workflow_engine/workflow_processor.py @@ -11,10 +11,9 @@ from pathlib import Path from itertools import product +from .logging.logger import logger from .schema_validator import SchemaValidator -from .task import * -from .utils import logger - +from .task import Task, TASKS_HANDLERS class WorkflowFile: @@ -272,20 +271,21 @@ def execute_task(self, dag: DAG, task: dict, action) -> None: """Execute a task.""" task_name = task['task'] if dag.should_be_canceled(task_name): - logger.warning("[%s] Skipping task due to dependency failure.", task_name) + logger.warning(f'Skipping task "{task_name}" due to dependency failure.') dag.set_status(task_name, 'canceled') else: try: task_object = self.create_task_object(task, action) - logger.info("[%s] Starting task.", task_name) + logger.info(f'Initiating execution of task "{task_name}".') start_time = time.time() task_object.execute() - logger.info("[%s] Finished task in %.2f seconds.", task_name, time.time() - start_time) + finish_time = (time.time() - start_time) + logger.info(f'Task "{task_name}" completed succesfully in {finish_time:.2f} seconds.') dag.set_status(task_name, 'successful') except Exception as e: # Pass the tag to the tag_formatter function if it exists - logger.error("[%s] Task failed with error: %s.", task_name, e) + logger.error(f'Task "{task_name}" failed with error: {e}.') dag.set_status(task_name, 'failed') dag.cancel_dependant_tasks(task_name, task.get('on-error', 'abort-related-flows')) # Handle the exception or re-raise if necessary @@ -294,21 +294,22 @@ def execute_task(self, dag: DAG, task: dict, action) -> None: def create_task_object(self, task: dict, action) -> Task: """Create and return a Task object based on task type.""" task_type = task[action]['this'] - task_handler = TASKS_HANDLERS.get(task_type) if task_handler is not None: return task_handler(task['task'], task[action]['with']) - raise ValueError(f"Unknown task type '{task_type}'.") + raise ValueError(f'Unknown task type "{task_type}".') def execute_tasks_parallel(self) -> None: """Execute tasks in parallel.""" if not self.dry_run: - logger.info("Executing tasks in parallel.") + logger.info('Starting workflow execution.') dag = DAG(self.task_collection) # Execute tasks based on the DAG with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: + if self.threads > 1: + logger.info(f'Running tasks in parallel using {self.threads} threads.') futures = {} while True: if not dag.is_active(): @@ -322,7 +323,7 @@ def execute_tasks_parallel(self) -> None: # Now execute cleanup tasks based on the reverse DAG reverse_dag = DAG(self.task_collection, reverse=True) - logger.info("Executing cleanup tasks.") + logger.info('Executing cleanup tasks.') with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: reverse_futures = {} @@ -340,7 +341,7 @@ def execute_tasks_parallel(self) -> None: else: dag = DAG(self.task_collection) - logger.info("Execution plan: %s", json.dumps(dag.get_execution_plan(), indent=2)) + logger.info(f'Execution plan: {json.dumps(dag.get_execution_plan())}', indent=2) def run(self) -> None: """Main entry point.""" @@ -352,6 +353,6 @@ def abort_execution(self, executor: concurrent.futures.ThreadPoolExecutor, futur try: _ = future.result() except Exception as e: - logger.error("Error in aborted task: %s", e) + logger.error(f"Error in aborted task: {e}") executor.shutdown(wait=False) From 32aaa80625a8fddf18a55195107bea8f81e78196 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Wed, 7 Feb 2024 11:09:56 -0300 Subject: [PATCH 03/23] Update deployability/modules/workflow_engine/logging/logger.py --- deployability/modules/workflow_engine/logging/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deployability/modules/workflow_engine/logging/logger.py b/deployability/modules/workflow_engine/logging/logger.py index 2cae892a1b..0a59d09459 100644 --- a/deployability/modules/workflow_engine/logging/logger.py +++ b/deployability/modules/workflow_engine/logging/logger.py @@ -21,7 +21,6 @@ def _load_config() -> None: _load_config() - # self.logger = logging.getLogger(name) logger = logging.getLogger("workflow_engine") # def parse_multiline_log(logger, log_message): From dc1729488f65f2eec7b735fe9154b20cbaafd76a Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Wed, 7 Feb 2024 11:10:07 -0300 Subject: [PATCH 04/23] Update deployability/modules/workflow_engine/logging/logger.py --- deployability/modules/workflow_engine/logging/logger.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deployability/modules/workflow_engine/logging/logger.py b/deployability/modules/workflow_engine/logging/logger.py index 0a59d09459..990ce06b9c 100644 --- a/deployability/modules/workflow_engine/logging/logger.py +++ b/deployability/modules/workflow_engine/logging/logger.py @@ -5,10 +5,6 @@ import yaml -# The logging module will use the generic logger anyway -# that is because the logging module is a singleton. -# So, we can just use it this way here, and it will work. -# formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(process)d] [%(threadName)s] %(name)s: %(message)s') def _load_config() -> None: """ From a60c1a5551163c719a4224d190fea25701d295ae Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Wed, 7 Feb 2024 11:10:16 -0300 Subject: [PATCH 05/23] Update deployability/modules/workflow_engine/logging/logger.py --- .../modules/workflow_engine/logging/logger.py | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/deployability/modules/workflow_engine/logging/logger.py b/deployability/modules/workflow_engine/logging/logger.py index 990ce06b9c..a3d46cb1a6 100644 --- a/deployability/modules/workflow_engine/logging/logger.py +++ b/deployability/modules/workflow_engine/logging/logger.py @@ -19,46 +19,3 @@ def _load_config() -> None: logger = logging.getLogger("workflow_engine") -# def parse_multiline_log(logger, log_message): -# """ -# Parses a multiline log message and logs it with the provided logger. - -# Args: -# logger (logging.Logger): The logger instance. -# log_message (str): The log message to parse and log. -# """ -# for line in log_message.split('\n'): -# if line: -# parse_and_log(logger, line) - -# # log_utils.py -# import logging -# import os -# import threading - -# def setup_logger(name): -# logger = logging.getLogger(name) -# handler = logging.StreamHandler() -# formatter = logging.Formatter('%(asctime)s [PID: %(process)d] [%(threadName)s] [%(levelname)s] %(message)s') -# handler.setFormatter(formatter) -# logger.addHandler(handler) -# logger.setLevel(logging.DEBUG) -# return logger - -# def parse_and_log(logger: logging.Logger, line: str, extra: dict = None): -# # Parse the log message -# try: -# _, log_level, log_message = line.split('[', 2) -# log_message = log_message.rstrip(']') -# except ValueError: -# log_message = line -# log_level = 'DEBUG' - -# if log_level.strip().upper() == 'DEBUG': -# logger.debug(log_message, extra=extra) -# elif log_level.strip().upper() == 'INFO': -# logger.info(log_message, extra=extra) -# elif log_level.strip().upper() == 'WARNING': -# logger.warning(log_message, extra=extra) -# elif log_level.strip().upper() == 'ERROR': -# logger.error(log_message, extra=extra) From 506d3150e5b1da8e0f6be68362de6c4aeda2deac Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 7 Feb 2024 15:59:53 +0100 Subject: [PATCH 06/23] enhacement(#4909): Changes in playbooks dir, setup.py and launchers --- deployability/Jenkinsfiles/Launcher.groovy | 5 +- deployability/Jenkinsfiles/Provision.groovy | 6 +-- deployability/launchers/__init__.py | 0 .../allocation/main.py} | 2 +- deployability/modules/generic/ansible.py | 6 ++- .../provision/main.py} | 2 +- .../provision/playbooks}/deps/dependencies.j2 | 0 .../playbooks}/generic/install/install.j2 | 0 .../playbooks}/generic/uninstall/uninstall.j2 | 0 .../playbooks}/information/host.yaml | 0 .../playbooks}/wazuh/aio/install/download.j2 | 0 .../playbooks}/wazuh/aio/install/install.j2 | 0 .../wazuh/aio/uninstall/download.j2 | 0 .../wazuh/aio/uninstall/uninstall.j2 | 0 .../wazuh/package/install/install.j2 | 0 .../wazuh/package/install/register.j2 | 0 .../wazuh/package/install/service.j2 | 0 .../wazuh/package/install/set_repo.j2 | 0 .../wazuh/package/uninstall/uninstall.j2 | 0 deployability/modules/setup.py | 49 +++++++++++++++++++ .../test.py => modules/testing/main.py} | 2 +- .../testing/playbooks}/cleanup.yml | 0 .../testing/playbooks}/setup.yml | 0 .../testing/playbooks}/test.yml | 0 deployability/modules/testing/testing.py | 6 +-- .../modules/workflow_engine.egg-info/PKG-INFO | 8 +++ .../workflow_engine.egg-info/SOURCES.txt | 29 +++++++++++ .../dependency_links.txt | 1 + .../workflow_engine.egg-info/entry_points.txt | 7 +++ .../workflow_engine.egg-info/not-zip-safe | 1 + .../workflow_engine.egg-info/top_level.txt | 1 + .../modules/workflow_engine/__init__.py | 2 +- .../workflow_engine/main.py} | 6 +-- .../workflow_engine/schema_validator.py | 2 +- deployability/modules/workflow_engine/task.py | 2 +- .../workflow_engine/workflow_processor.py | 6 +-- deployability/version.json | 4 ++ 37 files changed, 123 insertions(+), 24 deletions(-) delete mode 100755 deployability/launchers/__init__.py rename deployability/{launchers/allocation.py => modules/allocation/main.py} (98%) rename deployability/{launchers/provision.py => modules/provision/main.py} (98%) rename deployability/{playbooks/provision => modules/provision/playbooks}/deps/dependencies.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/generic/install/install.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/generic/uninstall/uninstall.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/information/host.yaml (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/aio/install/download.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/aio/install/install.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/aio/uninstall/download.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/aio/uninstall/uninstall.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/package/install/install.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/package/install/register.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/package/install/service.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/package/install/set_repo.j2 (100%) rename deployability/{playbooks/provision => modules/provision/playbooks}/wazuh/package/uninstall/uninstall.j2 (100%) create mode 100644 deployability/modules/setup.py rename deployability/{launchers/test.py => modules/testing/main.py} (98%) rename deployability/{playbooks/tests => modules/testing/playbooks}/cleanup.yml (100%) rename deployability/{playbooks/tests => modules/testing/playbooks}/setup.yml (100%) rename deployability/{playbooks/tests => modules/testing/playbooks}/test.yml (100%) create mode 100644 deployability/modules/workflow_engine.egg-info/PKG-INFO create mode 100644 deployability/modules/workflow_engine.egg-info/SOURCES.txt create mode 100644 deployability/modules/workflow_engine.egg-info/dependency_links.txt create mode 100644 deployability/modules/workflow_engine.egg-info/entry_points.txt create mode 100644 deployability/modules/workflow_engine.egg-info/not-zip-safe create mode 100644 deployability/modules/workflow_engine.egg-info/top_level.txt rename deployability/{launchers/workflow_engine.py => modules/workflow_engine/main.py} (90%) create mode 100644 deployability/version.json diff --git a/deployability/Jenkinsfiles/Launcher.groovy b/deployability/Jenkinsfiles/Launcher.groovy index fa5ada17ec..75e88108e8 100755 --- a/deployability/Jenkinsfiles/Launcher.groovy +++ b/deployability/Jenkinsfiles/Launcher.groovy @@ -1,7 +1,6 @@ - String jenkins_reference = params.getOrDefault('JENKINS_REFERENCE', 'enhancement/4751-dtt1-iteration-2-poc') -String launcher_path = "launchers" -String task_flow_launcher = "provision.py" +String launcher_path = "modules/provision" +String task_flow_launcher = "main.py" String workflow = "modules/workflow_engine/examples/dtt1-managers.yaml" String schema = "modules/workflow_engine/schema.json" diff --git a/deployability/Jenkinsfiles/Provision.groovy b/deployability/Jenkinsfiles/Provision.groovy index bb3aa4623b..eb50ddc6dc 100755 --- a/deployability/Jenkinsfiles/Provision.groovy +++ b/deployability/Jenkinsfiles/Provision.groovy @@ -1,7 +1,5 @@ - - -String provision_path = "${WORKSPACE}/scripts/provision" -String provision_script = "provision.py" +String provision_path = "${WORKSPACE}/modules/provision" +String provision_script = "main.py" String inventory = "inventory.yaml" String jenkins_reference = params.getOrDefault('JENKINS_REFERENCE', 'enhancement/4665-dtt1-poc') diff --git a/deployability/launchers/__init__.py b/deployability/launchers/__init__.py deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/deployability/launchers/allocation.py b/deployability/modules/allocation/main.py similarity index 98% rename from deployability/launchers/allocation.py rename to deployability/modules/allocation/main.py index 4540cd0ff3..cf6b60b1bb 100755 --- a/deployability/launchers/allocation.py +++ b/deployability/modules/allocation/main.py @@ -2,7 +2,7 @@ import os import sys -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) sys.path.append(project_root) from modules.allocation import Allocator diff --git a/deployability/modules/generic/ansible.py b/deployability/modules/generic/ansible.py index 590cfc5773..6d387af7fb 100755 --- a/deployability/modules/generic/ansible.py +++ b/deployability/modules/generic/ansible.py @@ -19,7 +19,9 @@ class Inventory(BaseModel): class Ansible: def __init__(self, ansible_data: dict | Inventory, path: str | Path = None): self.path = path - self.playbooks_path = Path(__file__).parents[2] / 'playbooks' + self.modules_path = Path(__file__).parents[2] + self.provision_playbook_path = self.modules_path / 'provision/playbooks' + self.testing_playbook_path = self.modules_path / 'testing/playbooks' self.ansible_data = Inventory(**dict(ansible_data)) self.inventory = self.generate_inventory() self.logger = Logger(Path(__file__).stem).get_logger() @@ -32,7 +34,7 @@ def render_playbooks(self, rendering_variables: dict) -> list[str]: rendering_variables (dict): Extra variables to render the playbooks. """ tasks = [] - path_to_render_playbooks = self.playbooks_path / rendering_variables.get("templates_path") + path_to_render_playbooks = self.provision_playbook_path / rendering_variables.get("templates_path") template_loader = jinja2.FileSystemLoader(searchpath=path_to_render_playbooks) template_env = jinja2.Environment(loader=template_loader) diff --git a/deployability/launchers/provision.py b/deployability/modules/provision/main.py similarity index 98% rename from deployability/launchers/provision.py rename to deployability/modules/provision/main.py index 161d8ce179..a222680f78 100755 --- a/deployability/launchers/provision.py +++ b/deployability/modules/provision/main.py @@ -6,7 +6,7 @@ # ---------------- Vars ------------------------ -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) sys.path.append(project_root) from modules.provision import Provision, models diff --git a/deployability/playbooks/provision/deps/dependencies.j2 b/deployability/modules/provision/playbooks/deps/dependencies.j2 similarity index 100% rename from deployability/playbooks/provision/deps/dependencies.j2 rename to deployability/modules/provision/playbooks/deps/dependencies.j2 diff --git a/deployability/playbooks/provision/generic/install/install.j2 b/deployability/modules/provision/playbooks/generic/install/install.j2 similarity index 100% rename from deployability/playbooks/provision/generic/install/install.j2 rename to deployability/modules/provision/playbooks/generic/install/install.j2 diff --git a/deployability/playbooks/provision/generic/uninstall/uninstall.j2 b/deployability/modules/provision/playbooks/generic/uninstall/uninstall.j2 similarity index 100% rename from deployability/playbooks/provision/generic/uninstall/uninstall.j2 rename to deployability/modules/provision/playbooks/generic/uninstall/uninstall.j2 diff --git a/deployability/playbooks/provision/information/host.yaml b/deployability/modules/provision/playbooks/information/host.yaml similarity index 100% rename from deployability/playbooks/provision/information/host.yaml rename to deployability/modules/provision/playbooks/information/host.yaml diff --git a/deployability/playbooks/provision/wazuh/aio/install/download.j2 b/deployability/modules/provision/playbooks/wazuh/aio/install/download.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/aio/install/download.j2 rename to deployability/modules/provision/playbooks/wazuh/aio/install/download.j2 diff --git a/deployability/playbooks/provision/wazuh/aio/install/install.j2 b/deployability/modules/provision/playbooks/wazuh/aio/install/install.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/aio/install/install.j2 rename to deployability/modules/provision/playbooks/wazuh/aio/install/install.j2 diff --git a/deployability/playbooks/provision/wazuh/aio/uninstall/download.j2 b/deployability/modules/provision/playbooks/wazuh/aio/uninstall/download.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/aio/uninstall/download.j2 rename to deployability/modules/provision/playbooks/wazuh/aio/uninstall/download.j2 diff --git a/deployability/playbooks/provision/wazuh/aio/uninstall/uninstall.j2 b/deployability/modules/provision/playbooks/wazuh/aio/uninstall/uninstall.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/aio/uninstall/uninstall.j2 rename to deployability/modules/provision/playbooks/wazuh/aio/uninstall/uninstall.j2 diff --git a/deployability/playbooks/provision/wazuh/package/install/install.j2 b/deployability/modules/provision/playbooks/wazuh/package/install/install.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/package/install/install.j2 rename to deployability/modules/provision/playbooks/wazuh/package/install/install.j2 diff --git a/deployability/playbooks/provision/wazuh/package/install/register.j2 b/deployability/modules/provision/playbooks/wazuh/package/install/register.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/package/install/register.j2 rename to deployability/modules/provision/playbooks/wazuh/package/install/register.j2 diff --git a/deployability/playbooks/provision/wazuh/package/install/service.j2 b/deployability/modules/provision/playbooks/wazuh/package/install/service.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/package/install/service.j2 rename to deployability/modules/provision/playbooks/wazuh/package/install/service.j2 diff --git a/deployability/playbooks/provision/wazuh/package/install/set_repo.j2 b/deployability/modules/provision/playbooks/wazuh/package/install/set_repo.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/package/install/set_repo.j2 rename to deployability/modules/provision/playbooks/wazuh/package/install/set_repo.j2 diff --git a/deployability/playbooks/provision/wazuh/package/uninstall/uninstall.j2 b/deployability/modules/provision/playbooks/wazuh/package/uninstall/uninstall.j2 similarity index 100% rename from deployability/playbooks/provision/wazuh/package/uninstall/uninstall.j2 rename to deployability/modules/provision/playbooks/wazuh/package/uninstall/uninstall.j2 diff --git a/deployability/modules/setup.py b/deployability/modules/setup.py new file mode 100644 index 0000000000..e6c6a68fa1 --- /dev/null +++ b/deployability/modules/setup.py @@ -0,0 +1,49 @@ + +# Copyright (C) 2015-2024, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +import json +from setuptools import setup, find_packages +import os + +def get_files_from_directory(directory): + paths = [] + for (path, directories, filenames) in os.walk(directory): + for filename in filenames: + paths.append(os.path.join('..', path, filename)) + return paths + +def get_version(): + script_path = os.path.dirname(__file__) + rel_path = "../version.json" + abs_file_path = os.path.join(script_path, rel_path) + f = open(abs_file_path) + data = json.load(f) + version = data['version'] + return version + +scripts_list = [ + 'models=workflow_engine.models:main', + 'schema_validator=workflow_engine.schema_validator:main', + 'task=workflow_engine.task:main', + 'workflow_processor=workflow_engine.workflow_processor:main', + 'main=workflow_engine.main:main' +] + +package_data_list = get_files_from_directory("workflow_engine") + +setup( + name='workflow_engine', + version=get_version(), + description='Wazuh testing utilities to help programmers automate deployment tests', + url='https://github.com/wazuh', + author='Wazuh', + author_email='hello@wazuh.com', + license='GPLv2', + packages=['workflow_engine'], + package_dir={'workflow_engine': 'workflow_engine'}, + package_data={'workflow_engine': package_data_list}, + entry_points={'console_scripts': scripts_list}, + include_package_data=True, + zip_safe=False +) \ No newline at end of file diff --git a/deployability/launchers/test.py b/deployability/modules/testing/main.py similarity index 98% rename from deployability/launchers/test.py rename to deployability/modules/testing/main.py index 3c2255c532..d7fef49089 100755 --- a/deployability/launchers/test.py +++ b/deployability/modules/testing/main.py @@ -2,7 +2,7 @@ import sys import os -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) sys.path.append(project_root) from modules.testing import Tester, InputPayload diff --git a/deployability/playbooks/tests/cleanup.yml b/deployability/modules/testing/playbooks/cleanup.yml similarity index 100% rename from deployability/playbooks/tests/cleanup.yml rename to deployability/modules/testing/playbooks/cleanup.yml diff --git a/deployability/playbooks/tests/setup.yml b/deployability/modules/testing/playbooks/setup.yml similarity index 100% rename from deployability/playbooks/tests/setup.yml rename to deployability/modules/testing/playbooks/setup.yml diff --git a/deployability/playbooks/tests/test.yml b/deployability/modules/testing/playbooks/test.yml similarity index 100% rename from deployability/playbooks/tests/test.yml rename to deployability/modules/testing/playbooks/test.yml diff --git a/deployability/modules/testing/testing.py b/deployability/modules/testing/testing.py index 204616d7ae..089dc6ea29 100644 --- a/deployability/modules/testing/testing.py +++ b/deployability/modules/testing/testing.py @@ -77,7 +77,7 @@ def _run_tests(cls, test_list: list[str], ansible: Ansible, extra_vars: ExtraVar """ for test in test_list: rendering_var = {**extra_vars, 'test': test} - template = str(ansible.playbooks_path / cls._test_template) + template = str(ansible.testing_playbook_path / cls._test_template) playbook = ansible.render_playbook(template, rendering_var) if not playbook: logger.warning(f"Test {test} not found. Skipped.") @@ -95,7 +95,7 @@ def _setup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: """ extra_vars = {'local_path': str(Path(__file__).parent / 'tests'), 'working_dir': remote_working_dir} - playbook = str(ansible.playbooks_path / cls._setup_playbook) + playbook = str(ansible.testing_playbook_path / cls._setup_playbook) ansible.run_playbook(playbook, extra_vars) @classmethod @@ -108,5 +108,5 @@ def _cleanup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: remote_working_dir (str): The remote working directory. """ extra_vars = {'working_dir': remote_working_dir} - playbook = str(ansible.playbooks_path / cls._cleanup_playbook) + playbook = str(ansible.testing_playbook_path / cls._cleanup_playbook) ansible.run_playbook(playbook, extra_vars) diff --git a/deployability/modules/workflow_engine.egg-info/PKG-INFO b/deployability/modules/workflow_engine.egg-info/PKG-INFO new file mode 100644 index 0000000000..7bc805e400 --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/PKG-INFO @@ -0,0 +1,8 @@ +Metadata-Version: 2.1 +Name: workflow_engine +Version: 1.0 +Summary: Wazuh testing utilities to help programmers automate deployment tests +Home-page: https://github.com/wazuh +Author: Wazuh +Author-email: hello@wazuh.com +License: GPLv2 diff --git a/deployability/modules/workflow_engine.egg-info/SOURCES.txt b/deployability/modules/workflow_engine.egg-info/SOURCES.txt new file mode 100644 index 0000000000..9d209788c1 --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/SOURCES.txt @@ -0,0 +1,29 @@ +setup.py +workflow_engine/README.md +workflow_engine/__init__.py +workflow_engine/main.py +workflow_engine/models.py +workflow_engine/schema_validator.py +workflow_engine/task.py +workflow_engine/workflow_processor.py +workflow_engine.egg-info/PKG-INFO +workflow_engine.egg-info/SOURCES.txt +workflow_engine.egg-info/dependency_links.txt +workflow_engine.egg-info/entry_points.txt +workflow_engine.egg-info/not-zip-safe +workflow_engine.egg-info/top_level.txt +workflow_engine/__pycache__/__init__.cpython-310.pyc +workflow_engine/__pycache__/models.cpython-310.pyc +workflow_engine/__pycache__/schema_validator.cpython-310.pyc +workflow_engine/__pycache__/task.cpython-310.pyc +workflow_engine/__pycache__/utils.cpython-310.pyc +workflow_engine/__pycache__/workflow_processor.cpython-310.pyc +workflow_engine/examples/dtt1-agents-poc.yaml +workflow_engine/examples/dtt1-agents.yaml +workflow_engine/examples/dtt1-managers.yaml +workflow_engine/logging/__init__.py +workflow_engine/logging/config.yaml +workflow_engine/logging/filter.py +workflow_engine/logging/logger.py +workflow_engine/logging/__pycache__/__init__.cpython-310.pyc +workflow_engine/schemas/schema_v1.json \ No newline at end of file diff --git a/deployability/modules/workflow_engine.egg-info/dependency_links.txt b/deployability/modules/workflow_engine.egg-info/dependency_links.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/deployability/modules/workflow_engine.egg-info/entry_points.txt b/deployability/modules/workflow_engine.egg-info/entry_points.txt new file mode 100644 index 0000000000..fbb14592ed --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/entry_points.txt @@ -0,0 +1,7 @@ +[console_scripts] +main = workflow_engine.main:main +models = workflow_engine.models:main +schema_validator = workflow_engine.schema_validator:main +task = workflow_engine.task:main +utils = workflow_engine.utils:main +workflow_processor = workflow_engine.workflow_processor:main diff --git a/deployability/modules/workflow_engine.egg-info/not-zip-safe b/deployability/modules/workflow_engine.egg-info/not-zip-safe new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/deployability/modules/workflow_engine.egg-info/top_level.txt b/deployability/modules/workflow_engine.egg-info/top_level.txt new file mode 100644 index 0000000000..16d139c485 --- /dev/null +++ b/deployability/modules/workflow_engine.egg-info/top_level.txt @@ -0,0 +1 @@ +workflow_engine diff --git a/deployability/modules/workflow_engine/__init__.py b/deployability/modules/workflow_engine/__init__.py index 338f58e9fd..1d21d4b0c4 100755 --- a/deployability/modules/workflow_engine/__init__.py +++ b/deployability/modules/workflow_engine/__init__.py @@ -1 +1 @@ -from .workflow_processor import WorkflowProcessor +#from .workflow_processor import WorkflowProcessor diff --git a/deployability/launchers/workflow_engine.py b/deployability/modules/workflow_engine/main.py similarity index 90% rename from deployability/launchers/workflow_engine.py rename to deployability/modules/workflow_engine/main.py index dc804bb34c..b298176b4e 100755 --- a/deployability/launchers/workflow_engine.py +++ b/deployability/modules/workflow_engine/main.py @@ -6,11 +6,11 @@ import sys import argparse -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) sys.path.append(project_root) -from modules.workflow_engine.workflow_processor import WorkflowProcessor -from modules.workflow_engine.models import InputPayload +from workflow_engine.workflow_processor import WorkflowProcessor +from workflow_engine.models import InputPayload def parse_arguments() -> argparse.Namespace: diff --git a/deployability/modules/workflow_engine/schema_validator.py b/deployability/modules/workflow_engine/schema_validator.py index 99b1db77a9..66b51b73ca 100755 --- a/deployability/modules/workflow_engine/schema_validator.py +++ b/deployability/modules/workflow_engine/schema_validator.py @@ -5,7 +5,7 @@ from pathlib import Path from ruamel.yaml import YAML -from .logging.logger import logger +from workflow_engine.logging.logger import logger class SchemaValidator: diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index e9695feed6..8eb1595c91 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -7,7 +7,7 @@ import time from abc import ABC, abstractmethod -from .logging.logger import logger +from workflow_engine.logging.logger import logger class Task(ABC): diff --git a/deployability/modules/workflow_engine/workflow_processor.py b/deployability/modules/workflow_engine/workflow_processor.py index f5e2be859a..0440ab2147 100755 --- a/deployability/modules/workflow_engine/workflow_processor.py +++ b/deployability/modules/workflow_engine/workflow_processor.py @@ -11,9 +11,9 @@ from pathlib import Path from itertools import product -from .logging.logger import logger -from .schema_validator import SchemaValidator -from .task import Task, TASKS_HANDLERS +from workflow_engine.logging.logger import logger +from workflow_engine.schema_validator import SchemaValidator +from workflow_engine.task import Task, TASKS_HANDLERS class WorkflowFile: diff --git a/deployability/version.json b/deployability/version.json new file mode 100644 index 0000000000..c885fdde40 --- /dev/null +++ b/deployability/version.json @@ -0,0 +1,4 @@ +{ + "version": "1.0", + "revision": "1" +} \ No newline at end of file From e40a2c9b4392db7999d7af4de9cb0ed0f7e96101 Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 7 Feb 2024 16:19:19 +0100 Subject: [PATCH 07/23] fix(#4909): Changes in script_list in setup --- deployability/modules/setup.py | 3 +- .../modules/workflow_engine.egg-info/PKG-INFO | 8 ----- .../workflow_engine.egg-info/SOURCES.txt | 29 ------------------- .../dependency_links.txt | 1 - .../workflow_engine.egg-info/entry_points.txt | 7 ----- .../workflow_engine.egg-info/not-zip-safe | 1 - .../workflow_engine.egg-info/top_level.txt | 1 - .../modules/workflow_engine/__init__.py | 2 +- 8 files changed, 3 insertions(+), 49 deletions(-) delete mode 100644 deployability/modules/workflow_engine.egg-info/PKG-INFO delete mode 100644 deployability/modules/workflow_engine.egg-info/SOURCES.txt delete mode 100644 deployability/modules/workflow_engine.egg-info/dependency_links.txt delete mode 100644 deployability/modules/workflow_engine.egg-info/entry_points.txt delete mode 100644 deployability/modules/workflow_engine.egg-info/not-zip-safe delete mode 100644 deployability/modules/workflow_engine.egg-info/top_level.txt diff --git a/deployability/modules/setup.py b/deployability/modules/setup.py index e6c6a68fa1..a0ae65eefb 100644 --- a/deployability/modules/setup.py +++ b/deployability/modules/setup.py @@ -27,7 +27,8 @@ def get_version(): 'schema_validator=workflow_engine.schema_validator:main', 'task=workflow_engine.task:main', 'workflow_processor=workflow_engine.workflow_processor:main', - 'main=workflow_engine.main:main' + 'main=workflow_engine.main:main', + 'logging=workflow_engine.logging.logger:main' ] package_data_list = get_files_from_directory("workflow_engine") diff --git a/deployability/modules/workflow_engine.egg-info/PKG-INFO b/deployability/modules/workflow_engine.egg-info/PKG-INFO deleted file mode 100644 index 7bc805e400..0000000000 --- a/deployability/modules/workflow_engine.egg-info/PKG-INFO +++ /dev/null @@ -1,8 +0,0 @@ -Metadata-Version: 2.1 -Name: workflow_engine -Version: 1.0 -Summary: Wazuh testing utilities to help programmers automate deployment tests -Home-page: https://github.com/wazuh -Author: Wazuh -Author-email: hello@wazuh.com -License: GPLv2 diff --git a/deployability/modules/workflow_engine.egg-info/SOURCES.txt b/deployability/modules/workflow_engine.egg-info/SOURCES.txt deleted file mode 100644 index 9d209788c1..0000000000 --- a/deployability/modules/workflow_engine.egg-info/SOURCES.txt +++ /dev/null @@ -1,29 +0,0 @@ -setup.py -workflow_engine/README.md -workflow_engine/__init__.py -workflow_engine/main.py -workflow_engine/models.py -workflow_engine/schema_validator.py -workflow_engine/task.py -workflow_engine/workflow_processor.py -workflow_engine.egg-info/PKG-INFO -workflow_engine.egg-info/SOURCES.txt -workflow_engine.egg-info/dependency_links.txt -workflow_engine.egg-info/entry_points.txt -workflow_engine.egg-info/not-zip-safe -workflow_engine.egg-info/top_level.txt -workflow_engine/__pycache__/__init__.cpython-310.pyc -workflow_engine/__pycache__/models.cpython-310.pyc -workflow_engine/__pycache__/schema_validator.cpython-310.pyc -workflow_engine/__pycache__/task.cpython-310.pyc -workflow_engine/__pycache__/utils.cpython-310.pyc -workflow_engine/__pycache__/workflow_processor.cpython-310.pyc -workflow_engine/examples/dtt1-agents-poc.yaml -workflow_engine/examples/dtt1-agents.yaml -workflow_engine/examples/dtt1-managers.yaml -workflow_engine/logging/__init__.py -workflow_engine/logging/config.yaml -workflow_engine/logging/filter.py -workflow_engine/logging/logger.py -workflow_engine/logging/__pycache__/__init__.cpython-310.pyc -workflow_engine/schemas/schema_v1.json \ No newline at end of file diff --git a/deployability/modules/workflow_engine.egg-info/dependency_links.txt b/deployability/modules/workflow_engine.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789179..0000000000 --- a/deployability/modules/workflow_engine.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deployability/modules/workflow_engine.egg-info/entry_points.txt b/deployability/modules/workflow_engine.egg-info/entry_points.txt deleted file mode 100644 index fbb14592ed..0000000000 --- a/deployability/modules/workflow_engine.egg-info/entry_points.txt +++ /dev/null @@ -1,7 +0,0 @@ -[console_scripts] -main = workflow_engine.main:main -models = workflow_engine.models:main -schema_validator = workflow_engine.schema_validator:main -task = workflow_engine.task:main -utils = workflow_engine.utils:main -workflow_processor = workflow_engine.workflow_processor:main diff --git a/deployability/modules/workflow_engine.egg-info/not-zip-safe b/deployability/modules/workflow_engine.egg-info/not-zip-safe deleted file mode 100644 index 8b13789179..0000000000 --- a/deployability/modules/workflow_engine.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deployability/modules/workflow_engine.egg-info/top_level.txt b/deployability/modules/workflow_engine.egg-info/top_level.txt deleted file mode 100644 index 16d139c485..0000000000 --- a/deployability/modules/workflow_engine.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -workflow_engine diff --git a/deployability/modules/workflow_engine/__init__.py b/deployability/modules/workflow_engine/__init__.py index 1d21d4b0c4..338f58e9fd 100755 --- a/deployability/modules/workflow_engine/__init__.py +++ b/deployability/modules/workflow_engine/__init__.py @@ -1 +1 @@ -#from .workflow_processor import WorkflowProcessor +from .workflow_processor import WorkflowProcessor From 5767f63ba0106c1a5d0e48dcfe4be37c343d4af1 Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 7 Feb 2024 17:52:53 +0100 Subject: [PATCH 08/23] enhancement(#4909): Chaning logging directory name --- deployability/modules/allocation/main.py | 5 ----- deployability/modules/provision/main.py | 3 --- deployability/modules/setup.py | 10 ---------- deployability/modules/testing/main.py | 3 --- .../workflow_engine/{logging => logger}/__init__.py | 0 .../workflow_engine/{logging => logger}/config.yaml | 0 .../workflow_engine/{logging => logger}/filter.py | 1 - .../workflow_engine/{logging => logger}/logger.py | 1 - deployability/modules/workflow_engine/main.py | 3 --- .../modules/workflow_engine/schema_validator.py | 3 +-- deployability/modules/workflow_engine/task.py | 7 +------ .../modules/workflow_engine/workflow_processor.py | 3 +-- 12 files changed, 3 insertions(+), 36 deletions(-) rename deployability/modules/workflow_engine/{logging => logger}/__init__.py (100%) rename deployability/modules/workflow_engine/{logging => logger}/config.yaml (100%) rename deployability/modules/workflow_engine/{logging => logger}/filter.py (99%) rename deployability/modules/workflow_engine/{logging => logger}/logger.py (99%) diff --git a/deployability/modules/allocation/main.py b/deployability/modules/allocation/main.py index cf6b60b1bb..8a69d380a7 100755 --- a/deployability/modules/allocation/main.py +++ b/deployability/modules/allocation/main.py @@ -2,9 +2,6 @@ import os import sys -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) -sys.path.append(project_root) - from modules.allocation import Allocator from modules.allocation.generic.models import InputPayload @@ -21,10 +18,8 @@ def parse_arguments(): parser.add_argument("--working-dir", required=False, default='/tmp/wazuh-qa') return parser.parse_args() - def main(): Allocator.run(InputPayload(**vars(parse_arguments()))) - if __name__ == "__main__": main() diff --git a/deployability/modules/provision/main.py b/deployability/modules/provision/main.py index a222680f78..355461923e 100755 --- a/deployability/modules/provision/main.py +++ b/deployability/modules/provision/main.py @@ -6,9 +6,6 @@ # ---------------- Vars ------------------------ -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) -sys.path.append(project_root) - from modules.provision import Provision, models # ---------------- Methods --------------------- diff --git a/deployability/modules/setup.py b/deployability/modules/setup.py index a0ae65eefb..982b78018c 100644 --- a/deployability/modules/setup.py +++ b/deployability/modules/setup.py @@ -22,15 +22,6 @@ def get_version(): version = data['version'] return version -scripts_list = [ - 'models=workflow_engine.models:main', - 'schema_validator=workflow_engine.schema_validator:main', - 'task=workflow_engine.task:main', - 'workflow_processor=workflow_engine.workflow_processor:main', - 'main=workflow_engine.main:main', - 'logging=workflow_engine.logging.logger:main' -] - package_data_list = get_files_from_directory("workflow_engine") setup( @@ -44,7 +35,6 @@ def get_version(): packages=['workflow_engine'], package_dir={'workflow_engine': 'workflow_engine'}, package_data={'workflow_engine': package_data_list}, - entry_points={'console_scripts': scripts_list}, include_package_data=True, zip_safe=False ) \ No newline at end of file diff --git a/deployability/modules/testing/main.py b/deployability/modules/testing/main.py index d7fef49089..10c0c3fe87 100755 --- a/deployability/modules/testing/main.py +++ b/deployability/modules/testing/main.py @@ -2,9 +2,6 @@ import sys import os -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) -sys.path.append(project_root) - from modules.testing import Tester, InputPayload def parse_arguments(): diff --git a/deployability/modules/workflow_engine/logging/__init__.py b/deployability/modules/workflow_engine/logger/__init__.py similarity index 100% rename from deployability/modules/workflow_engine/logging/__init__.py rename to deployability/modules/workflow_engine/logger/__init__.py diff --git a/deployability/modules/workflow_engine/logging/config.yaml b/deployability/modules/workflow_engine/logger/config.yaml similarity index 100% rename from deployability/modules/workflow_engine/logging/config.yaml rename to deployability/modules/workflow_engine/logger/config.yaml diff --git a/deployability/modules/workflow_engine/logging/filter.py b/deployability/modules/workflow_engine/logger/filter.py similarity index 99% rename from deployability/modules/workflow_engine/logging/filter.py rename to deployability/modules/workflow_engine/logger/filter.py index 9ef98fa179..099193a3ac 100644 --- a/deployability/modules/workflow_engine/logging/filter.py +++ b/deployability/modules/workflow_engine/logger/filter.py @@ -1,7 +1,6 @@ import logging import threading - class ThreadIDFilter(logging.Filter): """ A filter that uppercases the name of the log record. diff --git a/deployability/modules/workflow_engine/logging/logger.py b/deployability/modules/workflow_engine/logger/logger.py similarity index 99% rename from deployability/modules/workflow_engine/logging/logger.py rename to deployability/modules/workflow_engine/logger/logger.py index a3d46cb1a6..5922e6cc9a 100644 --- a/deployability/modules/workflow_engine/logging/logger.py +++ b/deployability/modules/workflow_engine/logger/logger.py @@ -5,7 +5,6 @@ import yaml - def _load_config() -> None: """ Loads the logging configuration from 'config.yaml' file. diff --git a/deployability/modules/workflow_engine/main.py b/deployability/modules/workflow_engine/main.py index b298176b4e..8a7e9e4f00 100755 --- a/deployability/modules/workflow_engine/main.py +++ b/deployability/modules/workflow_engine/main.py @@ -6,9 +6,6 @@ import sys import argparse -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) -sys.path.append(project_root) - from workflow_engine.workflow_processor import WorkflowProcessor from workflow_engine.models import InputPayload diff --git a/deployability/modules/workflow_engine/schema_validator.py b/deployability/modules/workflow_engine/schema_validator.py index 66b51b73ca..ae13a6da9f 100755 --- a/deployability/modules/workflow_engine/schema_validator.py +++ b/deployability/modules/workflow_engine/schema_validator.py @@ -5,8 +5,7 @@ from pathlib import Path from ruamel.yaml import YAML -from workflow_engine.logging.logger import logger - +from workflow_engine.logger.logger import logger class SchemaValidator: """ diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index 8eb1595c91..ea850f1e05 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -7,8 +7,7 @@ import time from abc import ABC, abstractmethod -from workflow_engine.logging.logger import logger - +from workflow_engine.logger.logger import logger class Task(ABC): """Abstract base class for tasks.""" @@ -17,8 +16,6 @@ class Task(ABC): def execute(self) -> None: """Execute the task.""" pass - - class ProcessTask(Task): """Task for executing a process.""" @@ -73,7 +70,6 @@ def execute(self): message = self.task_parameters.get('message', 'No message provided') logger.info("%s: %s", message, self.task_name, extra={'tag': self.task_name}) - class DummyRandomTask(Task): def __init__(self, task_name, task_parameters): self.task_name = task_name @@ -88,7 +84,6 @@ def execute(self): time.sleep(sleep_time) - TASKS_HANDLERS = { 'process': ProcessTask, 'dummy': DummyTask, diff --git a/deployability/modules/workflow_engine/workflow_processor.py b/deployability/modules/workflow_engine/workflow_processor.py index 0440ab2147..28fa4084b7 100755 --- a/deployability/modules/workflow_engine/workflow_processor.py +++ b/deployability/modules/workflow_engine/workflow_processor.py @@ -11,11 +11,10 @@ from pathlib import Path from itertools import product -from workflow_engine.logging.logger import logger +from workflow_engine.logger.logger import logger from workflow_engine.schema_validator import SchemaValidator from workflow_engine.task import Task, TASKS_HANDLERS - class WorkflowFile: """Class for loading and processing a workflow file.""" schema_path = Path(__file__).parent / 'schemas' / 'schema_v1.json' From b8c54da2cb84acbfff4bdc6087c3ff52a59e12fa Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 7 Feb 2024 19:47:37 +0100 Subject: [PATCH 09/23] fix(#4909): Fixing paths --- deployability/modules/allocation/main.py | 5 ++ deployability/modules/generic/ansible.py | 2 +- .../modules/provision/componentType.py | 8 +-- deployability/modules/provision/main.py | 3 ++ deployability/modules/setup.py | 2 + deployability/modules/testing/main.py | 3 ++ .../workflow_engine/{main.py => __main__.py} | 3 ++ .../examples/dtt1-agents-poc.yaml | 50 +++++++++---------- 8 files changed, 46 insertions(+), 30 deletions(-) rename deployability/modules/workflow_engine/{main.py => __main__.py} (92%) diff --git a/deployability/modules/allocation/main.py b/deployability/modules/allocation/main.py index 8a69d380a7..cf6b60b1bb 100755 --- a/deployability/modules/allocation/main.py +++ b/deployability/modules/allocation/main.py @@ -2,6 +2,9 @@ import os import sys +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) +sys.path.append(project_root) + from modules.allocation import Allocator from modules.allocation.generic.models import InputPayload @@ -18,8 +21,10 @@ def parse_arguments(): parser.add_argument("--working-dir", required=False, default='/tmp/wazuh-qa') return parser.parse_args() + def main(): Allocator.run(InputPayload(**vars(parse_arguments()))) + if __name__ == "__main__": main() diff --git a/deployability/modules/generic/ansible.py b/deployability/modules/generic/ansible.py index 6d387af7fb..e0ad09508b 100755 --- a/deployability/modules/generic/ansible.py +++ b/deployability/modules/generic/ansible.py @@ -19,7 +19,7 @@ class Inventory(BaseModel): class Ansible: def __init__(self, ansible_data: dict | Inventory, path: str | Path = None): self.path = path - self.modules_path = Path(__file__).parents[2] + self.modules_path = Path(__file__).parents[1] self.provision_playbook_path = self.modules_path / 'provision/playbooks' self.testing_playbook_path = self.modules_path / 'testing/playbooks' self.ansible_data = Inventory(**dict(ansible_data)) diff --git a/deployability/modules/provision/componentType.py b/deployability/modules/provision/componentType.py index 75f3bf4ae8..5295ba4fad 100644 --- a/deployability/modules/provision/componentType.py +++ b/deployability/modules/provision/componentType.py @@ -41,7 +41,7 @@ class Package(ComponentType): """ Class to define the type of package to be provisioned """ - TEMPLATE_BASE_PATH = 'provision/wazuh' + TEMPLATE_BASE_PATH = 'wazuh' def __init__(self, component_info, action): super().__init__(component_info) @@ -58,7 +58,7 @@ class AIO(ComponentType): """ Class to define the type of AIO to be provisioned """ - TEMPLATE_BASE_PATH = 'provision/wazuh' + TEMPLATE_BASE_PATH = 'wazuh' def __init__(self, component_info, action): super().__init__(component_info) @@ -73,7 +73,7 @@ class Generic(ComponentType): """ Class to define the type of generic component to be provisioned """ - TEMPLATE_BASE_PATH = 'provision/generic' + TEMPLATE_BASE_PATH = 'generic' def __init__(self, component_info, action): super().__init__(component_info) @@ -88,7 +88,7 @@ class Dependencies(ComponentType): """ Class to define the type of dependencies to be provisioned """ - TEMPLATE_BASE_PATH = 'provision/deps' + TEMPLATE_BASE_PATH = 'deps' def __init__(self, component_info, action): super().__init__(component_info) diff --git a/deployability/modules/provision/main.py b/deployability/modules/provision/main.py index 355461923e..a222680f78 100755 --- a/deployability/modules/provision/main.py +++ b/deployability/modules/provision/main.py @@ -6,6 +6,9 @@ # ---------------- Vars ------------------------ +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) +sys.path.append(project_root) + from modules.provision import Provision, models # ---------------- Methods --------------------- diff --git a/deployability/modules/setup.py b/deployability/modules/setup.py index 982b78018c..fea1915f39 100644 --- a/deployability/modules/setup.py +++ b/deployability/modules/setup.py @@ -23,6 +23,7 @@ def get_version(): return version package_data_list = get_files_from_directory("workflow_engine") +scripts_list = ['engine=workflow_engine.__main__:main'] setup( name='workflow_engine', @@ -35,6 +36,7 @@ def get_version(): packages=['workflow_engine'], package_dir={'workflow_engine': 'workflow_engine'}, package_data={'workflow_engine': package_data_list}, + entry_points={'console_scripts': scripts_list}, include_package_data=True, zip_safe=False ) \ No newline at end of file diff --git a/deployability/modules/testing/main.py b/deployability/modules/testing/main.py index 10c0c3fe87..d7fef49089 100755 --- a/deployability/modules/testing/main.py +++ b/deployability/modules/testing/main.py @@ -2,6 +2,9 @@ import sys import os +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) +sys.path.append(project_root) + from modules.testing import Tester, InputPayload def parse_arguments(): diff --git a/deployability/modules/workflow_engine/main.py b/deployability/modules/workflow_engine/__main__.py similarity index 92% rename from deployability/modules/workflow_engine/main.py rename to deployability/modules/workflow_engine/__main__.py index 8a7e9e4f00..b298176b4e 100755 --- a/deployability/modules/workflow_engine/main.py +++ b/deployability/modules/workflow_engine/__main__.py @@ -6,6 +6,9 @@ import sys import argparse +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) +sys.path.append(project_root) + from workflow_engine.workflow_processor import WorkflowProcessor from workflow_engine.models import InputPayload diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index f90f16c530..43f7624311 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -5,8 +5,8 @@ version: 0.1 description: This workflow is used to test agents deployment por DDT1 PoC variables: agents-os: - - linux-ubuntu-20.04-amd64 - manager-os: linux-ubuntu-20.04-amd64 + - linux-ubuntu-22.04-amd64 + manager-os: linux-ubuntu-22.04-amd64 infra-provider: vagrant working-dir: /tmp/dtt1-poc @@ -19,7 +19,7 @@ tasks: with: path: python3 args: - - test.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -43,7 +43,7 @@ tasks: with: path: python3 args: - - test.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -66,7 +66,7 @@ tasks: with: path: python3 args: - - provision.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: - component: wazuh-manager @@ -82,21 +82,21 @@ tasks: with: path: python3 args: - - allocation.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: large - composite-name: "{manager-os}" - inventory-output: "{working-dir}/manager-{manager-os}/inventory.yaml" - track-output: "{working-dir}/manager-{manager-os}/track.yaml" - cleanup: - this: process - with: - path: python3 - args: - - allocation.py - - action: delete - - track-output: "{working-dir}/manager-{manager-os}/track.yaml" + #cleanup: + # this: process + # with: + # path: python3 + # args: + # - /home/akim/Desktop/wazuh-qa/deployability/launchers/allocation.py + # - action: delete + # - track-output: "{working-dir}/manager-{manager-os}/track.yaml" # Generic agent provision task - task: "provision-install-{agent}" @@ -106,7 +106,7 @@ tasks: with: path: python3 args: - - provision.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -128,7 +128,7 @@ tasks: with: path: python3 args: - - provision.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -148,21 +148,21 @@ tasks: with: path: python3 args: - - allocation.py + - /home/akim/Desktop/wazuh-qa/deployability/modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: small - composite-name: "{agent}" - inventory-output: "{working-dir}/agent-{agent}/inventory.yaml" - track-output: "{working-dir}/agent-{agent}/track.yaml" - cleanup: - this: process - with: - path: python3 - args: - - allocation.py - - action: delete - - track-output: "{working-dir}/agent-{agent}/track.yaml" + #cleanup: + # this: process + # with: + # path: python3 + # args: + # - allocation.py + # - action: delete + # - track-output: "{working-dir}/agent-{agent}/track.yaml" foreach: - variable: agents-os as: agent \ No newline at end of file From 0c156f8f54936e208dd6d59aa7be28f4f0769e91 Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 8 Feb 2024 10:52:30 +0100 Subject: [PATCH 10/23] fix(#4909): Fixing paths and restoring dtt-1-agents-poc-yaml --- deployability/modules/generic/ansible.py | 1 - deployability/modules/testing/testing.py | 8 ++++---- .../examples/dtt1-agents-poc.yaml | 16 ++++++++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/deployability/modules/generic/ansible.py b/deployability/modules/generic/ansible.py index e0ad09508b..0d75fb1837 100755 --- a/deployability/modules/generic/ansible.py +++ b/deployability/modules/generic/ansible.py @@ -21,7 +21,6 @@ def __init__(self, ansible_data: dict | Inventory, path: str | Path = None): self.path = path self.modules_path = Path(__file__).parents[1] self.provision_playbook_path = self.modules_path / 'provision/playbooks' - self.testing_playbook_path = self.modules_path / 'testing/playbooks' self.ansible_data = Inventory(**dict(ansible_data)) self.inventory = self.generate_inventory() self.logger = Logger(Path(__file__).stem).get_logger() diff --git a/deployability/modules/testing/testing.py b/deployability/modules/testing/testing.py index 089dc6ea29..31cd9e5096 100644 --- a/deployability/modules/testing/testing.py +++ b/deployability/modules/testing/testing.py @@ -9,7 +9,7 @@ class Tester: - _playbooks_dir = Path('tests') + _playbooks_dir = Path('playbooks') _setup_playbook = _playbooks_dir / 'setup.yml' _cleanup_playbook = _playbooks_dir / 'cleanup.yml' _test_template = _playbooks_dir / 'test.yml' @@ -77,7 +77,7 @@ def _run_tests(cls, test_list: list[str], ansible: Ansible, extra_vars: ExtraVar """ for test in test_list: rendering_var = {**extra_vars, 'test': test} - template = str(ansible.testing_playbook_path / cls._test_template) + template = str(cls._test_template) playbook = ansible.render_playbook(template, rendering_var) if not playbook: logger.warning(f"Test {test} not found. Skipped.") @@ -95,7 +95,7 @@ def _setup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: """ extra_vars = {'local_path': str(Path(__file__).parent / 'tests'), 'working_dir': remote_working_dir} - playbook = str(ansible.testing_playbook_path / cls._setup_playbook) + playbook = str(cls._setup_playbook) ansible.run_playbook(playbook, extra_vars) @classmethod @@ -108,5 +108,5 @@ def _cleanup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: remote_working_dir (str): The remote working directory. """ extra_vars = {'working_dir': remote_working_dir} - playbook = str(ansible.testing_playbook_path / cls._cleanup_playbook) + playbook = str(cls._cleanup_playbook) ansible.run_playbook(playbook, extra_vars) diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index 43f7624311..bbe8cd19e4 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -19,7 +19,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/testing/main.py + - testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -43,7 +43,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/testing/main.py + - testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -66,7 +66,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py + - provision/main.py - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: - component: wazuh-manager @@ -82,7 +82,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/allocation/main.py + - allocation/main.py - action: create - provider: "{infra-provider}" - size: large @@ -94,7 +94,7 @@ tasks: # with: # path: python3 # args: - # - /home/akim/Desktop/wazuh-qa/deployability/launchers/allocation.py + # - wazuh-qa/deployability/launchers/allocation.py # - action: delete # - track-output: "{working-dir}/manager-{manager-os}/track.yaml" @@ -106,7 +106,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py + - provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -128,7 +128,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/provision/main.py + - provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -148,7 +148,7 @@ tasks: with: path: python3 args: - - /home/akim/Desktop/wazuh-qa/deployability/modules/allocation/main.py + - allocation/main.py - action: create - provider: "{infra-provider}" - size: small From af65f6180452ad248b2af0dcf996caec9cd9d768 Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 8 Feb 2024 10:57:08 +0100 Subject: [PATCH 11/23] fix(#4909): Fixing path testing.py --- deployability/modules/testing/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployability/modules/testing/testing.py b/deployability/modules/testing/testing.py index 31cd9e5096..a25f7ae8a8 100644 --- a/deployability/modules/testing/testing.py +++ b/deployability/modules/testing/testing.py @@ -7,9 +7,8 @@ from .models import InputPayload, ExtraVars from .utils import logger - class Tester: - _playbooks_dir = Path('playbooks') + _playbooks_dir = Path(__file__).parent / 'playbooks' _setup_playbook = _playbooks_dir / 'setup.yml' _cleanup_playbook = _playbooks_dir / 'cleanup.yml' _test_template = _playbooks_dir / 'test.yml' From b281a01279fe57db19cd45df625481e5162b1a97 Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 8 Feb 2024 10:58:39 +0100 Subject: [PATCH 12/23] fix(#4909): Fixing path testing --- deployability/modules/testing/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/testing/testing.py b/deployability/modules/testing/testing.py index a25f7ae8a8..04ddb29064 100644 --- a/deployability/modules/testing/testing.py +++ b/deployability/modules/testing/testing.py @@ -108,4 +108,4 @@ def _cleanup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: """ extra_vars = {'working_dir': remote_working_dir} playbook = str(cls._cleanup_playbook) - ansible.run_playbook(playbook, extra_vars) + ansible.run_playbook(playbook, extra_vars) \ No newline at end of file From f6588073d094c6aa75e1630feeeee4708147d69d Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 8 Feb 2024 11:01:19 +0100 Subject: [PATCH 13/23] fix(#4909): Fixing path in dtt1-agents-poc.yaml --- .../modules/workflow_engine/examples/dtt1-agents-poc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index bbe8cd19e4..faf9d1a18a 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -94,7 +94,7 @@ tasks: # with: # path: python3 # args: - # - wazuh-qa/deployability/launchers/allocation.py + # - allocation/main.py # - action: delete # - track-output: "{working-dir}/manager-{manager-os}/track.yaml" From f0d42a97db28afaf60f546460a96e0fba5b9e40e Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 8 Feb 2024 17:08:48 +0100 Subject: [PATCH 14/23] enhacement(#4906): Adding documentation --- .../modules/workflow_engine/README.md | 375 ++++++++++++++---- 1 file changed, 291 insertions(+), 84 deletions(-) diff --git a/deployability/modules/workflow_engine/README.md b/deployability/modules/workflow_engine/README.md index 74793771b3..b513275416 100755 --- a/deployability/modules/workflow_engine/README.md +++ b/deployability/modules/workflow_engine/README.md @@ -1,98 +1,305 @@ -# Workflow Processor - -The Workflow Processor is a tool for executing tasks defined in a YAML-based workflow file. It supports parallel execution of tasks with dependency management. - -## Table of Contents - -- [Workflow Processor](#workflow-processor) - - [Table of Contents](#table-of-contents) - - [Getting Started](#getting-started) - - [Prerequisites](#prerequisites) - - [Installation](#installation) - - [Usage](#usage) - - [Command Line Arguments](#command-line-arguments) - - [Workflow File](#workflow-file) - - [Logging](#logging) - - [Examples](#examples) - - [Basic Execution](#basic-execution) - - [Parallel Execution](#parallel-execution) - - [Dry Run](#dry-run) - - [License](#license) - -## Getting Started - -### Prerequisites - -Before using the Workflow Processor, make sure you have the following prerequisites installed: - -- Python 3.9 - -### Installation - -1. Clone the repository: - - ```bash - git clone https://github.com/wazuh/wazuh-qa.git - ``` - -2. Navigate to the project directory: - - ```bash - cd wazuh-qa/poc-tests/scripts/qa-workflow-engine - ``` - -3. Install the required dependencies: - - ```bash - pip install -r requirements.txt - ``` - -Now, you're ready to use the QA Workflow Engine. - -## Usage - -### Command Line Arguments - -Run the workflow processor using the following command: +## Workflow engine + +### User documentation + +The execution of the Workflow is done through the installation of its library. + +Initially, Python libraries must be installed. It is recommended to use virtual environments. Follow the technical documentation at https://docs.python.org/3/library/venv.html. + +1. Activate the environment: + + ```bash + source {venv directory}/bin/activate + ``` + +2. Clone the `wazuh-qa` repository: + + Navigate to the project directory and switch to the project branch: + + ```bash + cd wazuh-qa + git checkout {project-branch} + ``` + +3. Install requirements: + + ```bash + pip3 install -r /deployability/deps/requirements.txt + ``` + +4. Install the Workflow engine library and its launcher: + + While in wazuh-qa: + + ```bash + cd modules + pip3 uninstall -y workflow_engine && pip3 install . + ``` + +5. Test Fixture to Execute: + + It will be necessary to create a fixture (yaml file) where the infrastructure, provisioning, and tests to be executed will be declared. + + >Note: It is possible to find some fixture examples in deployability/modules/workflow_engine/examples/ + + Example: + + ```bash + version: 0.1 + description: This workflow is used to test agents deployment por DDT1 PoC + variables: + agents-os: + - linux-ubuntu-22.04-amd64 + manager-os: linux-ubuntu-22.04-amd64 + infra-provider: vagrant + working-dir: /tmp/dtt1-poc + + tasks: + # Generic agent test task + - task: "run-agent-tests-{agent}" + description: "Run tests uninstall for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - testing/main.py + - inventory: "{working-dir}/agent-{agent}/inventory.yaml" + - dependencies: + - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - agent: "{working-dir}/agent-{agent}/inventory.yaml" + - tests: "install,register,stop" + - component: "agent" + - wazuh-version: "4.7.1" + - wazuh-revision: "40709" + depends-on: + - "provision-install-{agent}" + - "provision-manager" + foreach: + - variable: agents-os + as: agent + + # Generic agent test task + - task: "run-agent-tests-uninstall-{agent}" + description: "Run tests uninstall for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - testing/main.py + - inventory: "{working-dir}/agent-{agent}/inventory.yaml" + - dependencies: + - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - tests: "uninstall" + - component: "agent" + - wazuh-version: "4.7.1" + - wazuh-revision: "40709" + depends-on: + - "run-agent-tests-{agent}" + - "provision-uninstall-{agent}" + foreach: + - variable: agents-os + as: agent + + # Unique manager allocate task + - task: "allocate-manager" + description: "Allocate resources for the manager." + do: + this: process + with: + path: python3 + args: + - allocation/main.py + - action: create + - provider: "{infra-provider}" + - size: large + - composite-name: "{manager-os}" + - inventory-output: "{working-dir}/manager-{manager-os}/inventory.yaml" + - track-output: "{working-dir}/manager-{manager-os}/track.yaml" + cleanup: + this: process + with: + path: python3 + args: + - allocation/main.py + - action: delete + - track-output: "{working-dir}/manager-{manager-os}/track.yaml" + + # Generic agent provision task + - task: "provision-install-{agent}" + description: "Provision resources for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - provision/main.py + - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" + - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - install: + - component: wazuh-agent + type: package + - component: curl + depends-on: + - "allocate-{agent}" + - "provision-manager" + foreach: + - variable: agents-os + as: agent + + # Generic agent provision task + - task: "provision-uninstall-{agent}" + description: "Provision resources for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - provision/main.py + - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" + - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - uninstall: + - component: wazuh-agent + type: package + depends-on: + - "provision-install-{agent}" + foreach: + - variable: agents-os + as: agent + + # Generic agent allocate task + - task: "allocate-{agent}" + description: "Allocate resources for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - allocation/main.py + - action: create + - provider: "{infra-provider}" + - size: small + - composite-name: "{agent}" + - inventory-output: "{working-dir}/agent-{agent}/inventory.yaml" + - track-output: "{working-dir}/agent-{agent}/track.yaml" + cleanup: + this: process + with: + path: python3 + args: + - allocation.py + - action: delete + - track-output: "{working-dir}/agent-{agent}/track.yaml" + foreach: + - variable: agents-os + as: agent + ``` + + Following the schema of the example: + + Configure the following parameters depending on your test case: + + ```yaml + variables/agent-os + variables/manager-os + infra-provider + working-dir + tasks + ``` + + Pay attention to the tasks: + + ```yaml + args + depends-on + ``` + + >Note: In args, configure the launcher's path correctly (main.py files in each module), and to fill `depends-on`, consider the steps of your test (allocation, provision, and test) + +7. Execution of Command (local): + + Execute the command by referencing the parameters required by the library (launcher). + + ```bash + python3 -m workflow_engine {.yaml fixture path} + ``` + + Example + + ```bash + python3 -m workflow_engine modules/workflow_engine/examples/dtt1-agents-poc.yaml + ``` + + > Note The command execution can also be mediated through Jenkins. + +--- + +### Technical documentation + +`Workflow Engine` is the orchestrator of the deployability test architecture. + +Its function is to allow the ordered and structured execution in steps of allocation, provision, and testing. + +`The Workflow Engine` receives instructions through a `YAML document`, the structure of which can be exemplified in tests found in: +`wazuh-qa/deployability/modules/workflow_engine/examples` + +**In these tests**: + - Tasks: define the steps. + - Task: defines a step. + +**Within Task**: + - description: description of the task. + - do: instructions for the task. + - this: nature of the task. + - with: tools with which the task will be executed. + - path: executable. + - args: arguments. it receives the binary or file to execute and the parameters. + - depends-on: steps prior to the execution of that task. + - foreach: loop that executes the task on the previously declared hosts. ```bash -python main.py workflow_file.yml --threads 4 --dry-run --log-format json --log-level INFO +tasks: + # Generic agent test task + - task: "run-agent-tests-{agent}" + description: "Run tests uninstall for the {agent} agent." + do: + this: process + with: + path: python3 + args: + - testing/main.py + - inventory: "{working-dir}/agent-{agent}/inventory.yaml" + - dependencies: + - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - agent: "{working-dir}/agent-{agent}/inventory.yaml" + - tests: "install,register,stop" + - component: "agent" + - wazuh-version: "4.7.1" + - wazuh-revision: "40709" + depends-on: + - "provision-install-{agent}" + - "provision-manager" + foreach: + - variable: agents-os + as: agent ``` -- `workflow_file.yml`: Path to the YAML-based workflow file. -- `--threads`: Number of threads to use for parallel execution (default is 1). -- `--dry-run`: Display the plan without executing tasks. -- `--log-format`: Log format (`plain` or `json`, default is `plain`). -- `--log-level`: Log level (`DEBUG`, `INFO`, `WARNING`, `ERROR`, or `CRITICAL`, default is `INFO`). - -### Workflow File - -The workflow file is written in YAML format. It defines tasks, dependencies, and other configurations. See the provided examples in the `examples/` directory for reference. - -### Logging +These tasks are executed by the `Workflow Engine` launcher installed as workflow_engine library in your virtual environment. -The workflow processor logs messages to the console. You can configure the log format (`plain` or `json`) and log level using command line arguments. +This launcher receives the parameters, sets up the test logs, and proceeds with the ordered execution. -## Examples +The parameters sent from the launcher are processed by deployability/modules/workflow_engine/models.py, which checks the nature of the parameters sent and filters out incorrect parameters. -### Basic Execution +![image](https://github.com/wazuh/wazuh-qa/assets/125690423/32aa77b7-f294-41ac-af93-db8a084dbad1) -```bash -python main.py examples/basic_workflow.yml -``` +These are then sent to `deployability/modules/workflow_engine/workflow_processor.py`, where using `deployability/modules/schemas`, instructions in YAML are received and the schema of the instructions is checked. -### Parallel Execution +The commands are executed in the WorkflowProcessor of the same file, which also handles parallel executions and aborts failed executions. -```bash -python main.py examples/parallel_workflow.yml --threads 4 -``` +[WF.drawio.zip](https://github.com/wazuh/wazuh-qa/files/14167559/WF.drawio.zip) -### Dry Run - -```bash -python main.py examples/dry_run_workflow.yml --dry-run -``` -## License +### License WAZUH Copyright (C) 2015 Wazuh Inc. (License GPLv2) From 2bdf001ea74dc3dfb9d43d66525396de3c9652ae Mon Sep 17 00:00:00 2001 From: fcaffieri Date: Wed, 14 Feb 2024 16:50:57 -0300 Subject: [PATCH 15/23] Improve error handling, add sigterm handle improve logging --- .gitignore | 6 +- .../allocation/static/templates/vagrant.j2 | 5 +- .../modules/allocation/vagrant/models.py | 1 + .../modules/allocation/vagrant/provider.py | 5 +- .../modules/workflow_engine/__main__.py | 14 +- .../examples/dtt1-agents-poc.yaml | 18 +-- .../workflow_engine/schema_validator.py | 8 + deployability/modules/workflow_engine/task.py | 17 ++- .../workflow_engine/workflow_processor.py | 139 +++++++++++------- 9 files changed, 135 insertions(+), 78 deletions(-) diff --git a/.gitignore b/.gitignore index bedbb9a72e..b81f4dcadf 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,8 @@ ansible.log key #Python -__pycache__/ \ No newline at end of file +__pycache__/ + +#Python setup +*.egg-info/ +build/ \ No newline at end of file diff --git a/deployability/modules/allocation/static/templates/vagrant.j2 b/deployability/modules/allocation/static/templates/vagrant.j2 index fa96aaf783..f151640214 100755 --- a/deployability/modules/allocation/static/templates/vagrant.j2 +++ b/deployability/modules/allocation/static/templates/vagrant.j2 @@ -8,8 +8,9 @@ Vagrant.configure("2") do |config| # VirtualBox specific settings config.vm.provider "virtualbox" do |v| - v.memory = {{ config.memory }} - v.cpus = {{ config.cpu }} + v.memory = "{{ config.memory }}" + v.cpus = "{{ config.cpu }}" + v.name = "{{ config.name }}" end # Network settings diff --git a/deployability/modules/allocation/vagrant/models.py b/deployability/modules/allocation/vagrant/models.py index 356b28defa..adf628b200 100644 --- a/deployability/modules/allocation/vagrant/models.py +++ b/deployability/modules/allocation/vagrant/models.py @@ -12,6 +12,7 @@ class VagrantConfig(ProviderConfig): box_version: str private_key: str public_key: str + name: str @field_validator('public_key', mode='before') @classmethod diff --git a/deployability/modules/allocation/vagrant/provider.py b/deployability/modules/allocation/vagrant/provider.py index 5585477f84..4f1c0f989c 100644 --- a/deployability/modules/allocation/vagrant/provider.py +++ b/deployability/modules/allocation/vagrant/provider.py @@ -45,7 +45,7 @@ def _create_instance(cls, base_dir: Path, params: CreationPayload, config: Vagra # Generate the credentials. credentials.generate(instance_dir, 'instance_key') # Parse the config if it is not provided. - config = cls.__parse_config(params, credentials) + config = cls.__parse_config(params, credentials, instance_id) else: logger.debug(f"Using provided config") credentials.load(config.public_key) @@ -119,7 +119,7 @@ def __render_vagrantfile(cls, config: VagrantConfig) -> str: return template.render(config=config) @classmethod - def __parse_config(cls, params: CreationPayload, credentials: VagrantCredentials) -> VagrantConfig: + def __parse_config(cls, params: CreationPayload, credentials: VagrantCredentials, instance_id: str) -> VagrantConfig: """ Parses the configuration for a Vagrant instance. @@ -142,6 +142,7 @@ def __parse_config(cls, params: CreationPayload, credentials: VagrantCredentials config['public_key'] = str(credentials.key_path.with_suffix('.pub')) config['cpu'] = size_specs['cpu'] config['memory'] = size_specs['memory'] + config['name'] = instance_id return VagrantConfig(**config) diff --git a/deployability/modules/workflow_engine/__main__.py b/deployability/modules/workflow_engine/__main__.py index b298176b4e..8f2dfab0a2 100755 --- a/deployability/modules/workflow_engine/__main__.py +++ b/deployability/modules/workflow_engine/__main__.py @@ -1,3 +1,4 @@ + # Copyright (C) 2015, Wazuh Inc. # Created by Wazuh, Inc. . # This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 @@ -5,6 +6,7 @@ import os import sys import argparse +import signal project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) sys.path.append(project_root) @@ -26,11 +28,13 @@ def parse_arguments() -> argparse.Namespace: def main() -> None: """Main entry point.""" - - args = parse_arguments() - processor = WorkflowProcessor(**dict(InputPayload(**vars(args)))) - processor.run() - + try: + args = parse_arguments() + processor = WorkflowProcessor(**dict(InputPayload(**vars(args)))) + signal.signal(signal.SIGINT, processor.handle_interrupt) + processor.run() + except Exception as e: + sys.exit(f"Error while provisioning: {e}") if __name__ == "__main__": main() diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index faf9d1a18a..d1b7572acd 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -21,7 +21,7 @@ tasks: args: - testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - - dependencies: + - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - agent: "{working-dir}/agent-{agent}/inventory.yaml" - tests: "install,register,stop" @@ -89,14 +89,14 @@ tasks: - composite-name: "{manager-os}" - inventory-output: "{working-dir}/manager-{manager-os}/inventory.yaml" - track-output: "{working-dir}/manager-{manager-os}/track.yaml" - #cleanup: - # this: process - # with: - # path: python3 - # args: - # - allocation/main.py - # - action: delete - # - track-output: "{working-dir}/manager-{manager-os}/track.yaml" + cleanup: + this: process + with: + path: python3 + args: + - allocation/main.py + - action: delete + - track-output: "{working-dir}/manager-{manager-os}/track.yaml" # Generic agent provision task - task: "provision-install-{agent}" diff --git a/deployability/modules/workflow_engine/schema_validator.py b/deployability/modules/workflow_engine/schema_validator.py index ae13a6da9f..7039ee76d9 100755 --- a/deployability/modules/workflow_engine/schema_validator.py +++ b/deployability/modules/workflow_engine/schema_validator.py @@ -1,5 +1,6 @@ import jsonschema import json +import os from jsonschema.exceptions import ValidationError from pathlib import Path @@ -28,10 +29,17 @@ def __init__(self, schema: Path | str, to_validate: Path | str): yaml_data: str = None self.logger = logger + + if not os.path.exists(schema): + raise FileNotFoundError(f'File "{schema}" not found.') + with open(schema, 'r') as schema_file: self.logger.debug(f"Loading schema file: {schema}") schema_data = json.load(schema_file) + if not os.path.exists(to_validate): + raise FileNotFoundError(f'File "{to_validate}" not found.') + with open(to_validate, 'r') as file: self.logger.debug(f"Loading yaml file: {to_validate}") yaml = YAML(typ='safe', pure=True) diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index ea850f1e05..80d568e845 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -16,6 +16,7 @@ class Task(ABC): def execute(self) -> None: """Execute the task.""" pass + class ProcessTask(Task): """Task for executing a process.""" @@ -36,6 +37,9 @@ def execute(self) -> None: """Execute the process task.""" task_args = [] + if self.task_parameters.get('args') is None: + raise ValueError(f'Not argument found in {self.task_name}') + for arg in self.task_parameters['args']: if isinstance(arg, str): task_args.append(arg) @@ -45,7 +49,11 @@ def execute(self) -> None: task_args.extend([f"--{key}={argvalue}" for argvalue in value]) else: task_args.append(f"--{key}={value}") - self.logger.debug(f'Running task "{self.task_name}" with arguments: {task_args}') + else: + logger.error(f'Coulg not parse arguments {arg}') + + logger.debug(f'Running task "{self.task_name}" with arguments: {task_args}') + result = None try: result = subprocess.run( @@ -56,9 +64,12 @@ def execute(self) -> None: ) logger.debug(f'Finished task "{self.task_name}" execution with result:\n{str(result.stdout)}') - if result.returncode != 0: - raise subprocess.CalledProcessError(returncode=result.returncode, cmd=result.args, output=result.stdout) + #if result.returncode != 0: + # raise subprocess.CalledProcessError(returncode=result.returncode, cmd=result.args, output=result.stdout) except subprocess.CalledProcessError as e: + error_msg = e.stderr + if "KeyboardInterrupt" in error_msg: + raise KeyboardInterrupt(f"Error executing process task with keyboard interrupt") raise Exception(f"Error executing process task {e.stderr}") class DummyTask(Task): diff --git a/deployability/modules/workflow_engine/workflow_processor.py b/deployability/modules/workflow_engine/workflow_processor.py index 28fa4084b7..6b8dd3d2f7 100755 --- a/deployability/modules/workflow_engine/workflow_processor.py +++ b/deployability/modules/workflow_engine/workflow_processor.py @@ -7,8 +7,9 @@ import json import time import yaml +import os -from pathlib import Path +from pathlib import Path from itertools import product from workflow_engine.logger.logger import logger @@ -33,9 +34,14 @@ def __validate_schema(self, workflow_file: Path | str) -> None: Args: workflow_file (Path | str): Path to the workflow file. """ - validator = SchemaValidator(self.schema_path, workflow_file) - validator.preprocess_data() - validator.validateSchema() + try: + logger.debug(f"Validating inptu file: {workflow_file}") + validator = SchemaValidator(self.schema_path, workflow_file) + validator.preprocess_data() + validator.validateSchema() + except Exception as e: + logger.error("Error while validating schema [%s] with error: %s", self.schema_path, e) + raise def __load_workflow(self, file_path: str) -> dict: """ @@ -47,15 +53,25 @@ def __load_workflow(self, file_path: str) -> dict: Returns: dict: Workflow data. """ + + if not os.path.exists(file_path): + raise FileNotFoundError(f'File "{file_path}" not found.') + + logger.debug(f"Loading workflow file: {file_path}") + with open(file_path, 'r', encoding='utf-8') as file: return yaml.safe_load(file) def __process_workflow(self): """Process the workflow and return a list of tasks.""" + logger.debug("Process workflow.") task_collection = [] variables = self.workflow_raw_data.get('variables', {}) for task in self.workflow_raw_data.get('tasks', []): task_collection.extend(self.__expand_task(task, variables)) + + if not task_collection: + raise ValueError("No tasks found in the workflow.") return task_collection def __replace_placeholders(self, element: str, values: dict, parent_key: str = None): @@ -270,88 +286,99 @@ def execute_task(self, dag: DAG, task: dict, action) -> None: """Execute a task.""" task_name = task['task'] if dag.should_be_canceled(task_name): - logger.warning(f'Skipping task "{task_name}" due to dependency failure.') + logger.warning("[%s] Skipping task due to dependency failure.", task_name) dag.set_status(task_name, 'canceled') else: try: task_object = self.create_task_object(task, action) - logger.info(f'Initiating execution of task "{task_name}".') + logger.info("[%s] Starting task.", task_name) start_time = time.time() task_object.execute() - finish_time = (time.time() - start_time) - logger.info(f'Task "{task_name}" completed succesfully in {finish_time:.2f} seconds.') + logger.info("[%s] Finished task in %.2f seconds.", task_name, time.time() - start_time) dag.set_status(task_name, 'successful') + except KeyboardInterrupt as e: + logger.error("[%s] Task failed with error: %s.", task_name, e) + dag.set_status(task_name, 'failed') + dag.cancel_dependant_tasks(task_name, task.get('on-error', 'abort-related-flows')) + raise KeyboardInterrupt except Exception as e: - # Pass the tag to the tag_formatter function if it exists - logger.error(f'Task "{task_name}" failed with error: {e}.') + logger.error("[%s] Task failed with error: %s.", task_name, e) dag.set_status(task_name, 'failed') dag.cancel_dependant_tasks(task_name, task.get('on-error', 'abort-related-flows')) - # Handle the exception or re-raise if necessary raise + def create_task_object(self, task: dict, action) -> Task: """Create and return a Task object based on task type.""" task_type = task[action]['this'] + task_handler = TASKS_HANDLERS.get(task_type) if task_handler is not None: return task_handler(task['task'], task[action]['with']) - raise ValueError(f'Unknown task type "{task_type}".') + raise ValueError(f"Unknown task type '{task_type}'.") - def execute_tasks_parallel(self) -> None: + def execute_tasks_parallel(self, dag: DAG, reverse: bool = False) -> None: """Execute tasks in parallel.""" - if not self.dry_run: - logger.info('Starting workflow execution.') - dag = DAG(self.task_collection) - # Execute tasks based on the DAG + logger.info("Executing tasks in parallel.") + try: with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: - if self.threads > 1: - logger.info(f'Running tasks in parallel using {self.threads} threads.') - futures = {} - while True: - if not dag.is_active(): - break - for task_name in dag.get_available_tasks(): - task = next(t for t in self.task_collection if t['task'] == task_name) - future = executor.submit(self.execute_task, dag, task, 'do') - futures[task_name] = future + futures = self.generate_futures(dag, executor, reverse) concurrent.futures.wait(futures.values()) + except KeyboardInterrupt: + logger.error("User interrupt detected. Aborting execution...") + self.execute_tasks_parallel(dag, reverse=True) - # Now execute cleanup tasks based on the reverse DAG - reverse_dag = DAG(self.task_collection, reverse=True) + def generate_futures(self, dag, executor, reverse: bool = False): + futures = {} - logger.info('Executing cleanup tasks.') - with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: - reverse_futures = {} - - while True: - if not reverse_dag.is_active(): - break - for task_name in reverse_dag.get_available_tasks(): - task = next(t for t in self.task_collection if t['task'] == task_name) - if 'cleanup' in task: - future = executor.submit(self.execute_task, reverse_dag, task, 'cleanup') - reverse_futures[task_name] = future - else: - reverse_dag.set_status(task_name, 'successful') - concurrent.futures.wait(reverse_futures.values()) + while True: + if not dag.is_active(): + break + + for task_name in list(dag.get_available_tasks()): + task = next(t for t in self.task_collection if t['task'] == task_name) + action = 'cleanup' if reverse and 'cleanup' in task else 'do' + if reverse and 'cleanup' not in task: + dag.set_status(task_name, 'successful') + else: + future = executor.submit(self.execute_task, dag, task, action) + futures[task_name] = future + + return futures - else: - dag = DAG(self.task_collection) - logger.info(f'Execution plan: {json.dumps(dag.get_execution_plan())}', indent=2) def run(self) -> None: """Main entry point.""" - self.execute_tasks_parallel() + try: + if not self.dry_run: + logger.info("Executing DAG tasks.") + dag = DAG(self.task_collection) + self.execute_tasks_parallel(dag) + + logger.info("Executing Reverse DAG tasks.") + reversed_dag = DAG(self.task_collection, reverse=True) + self.execute_tasks_parallel(reversed_dag, reverse=True) + else: + dag = DAG(self.task_collection) + logger.info("Execution plan: %s", json.dumps(dag.get_execution_plan(), indent=2)) + + except Exception as e: + logger.error("Error in Workflow: %s", e) - def abort_execution(self, executor: concurrent.futures.ThreadPoolExecutor, futures: dict) -> None: - """Abort the execution of tasks.""" - for future in concurrent.futures.as_completed(futures.values()): - try: - _ = future.result() - except Exception as e: - logger.error(f"Error in aborted task: {e}") - executor.shutdown(wait=False) + def handle_interrupt(self, signum, frame): + logger.error("User interrupt detected. End process...") + raise KeyboardInterrupt("User interrupt detected. End process...") + + def abort_execution(self, futures) -> None: + """Abort the execution of tasks.""" + with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor: + for future in concurrent.futures.as_completed(futures.values()): + try: + _ = future.result() + except Exception as e: + logger.error("Error in aborted task: %s", e) + executor.shutdown(wait=False, cancel_futures=True) From d8659006955a6b65f83e9cf30788978486c19695 Mon Sep 17 00:00:00 2001 From: fcaffieri Date: Thu, 15 Feb 2024 09:51:05 -0300 Subject: [PATCH 16/23] Improve error handling, add sigterm handle improve logging --- deployability/modules/workflow_engine/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index 80d568e845..410d4284bf 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -64,8 +64,8 @@ def execute(self) -> None: ) logger.debug(f'Finished task "{self.task_name}" execution with result:\n{str(result.stdout)}') - #if result.returncode != 0: - # raise subprocess.CalledProcessError(returncode=result.returncode, cmd=result.args, output=result.stdout) + if result.returncode != 0: + raise subprocess.CalledProcessError(returncode=result.returncode, cmd=result.args, output=result.stdout) except subprocess.CalledProcessError as e: error_msg = e.stderr if "KeyboardInterrupt" in error_msg: From ebce158066ecff2dc0cb82cc7f902dc54b6d489d Mon Sep 17 00:00:00 2001 From: fcaffieri Date: Thu, 15 Feb 2024 12:17:16 -0300 Subject: [PATCH 17/23] Fix review comments --- deployability/modules/allocation/allocation.py | 2 +- deployability/modules/allocation/aws/instance.py | 6 +++--- deployability/modules/workflow_engine/__main__.py | 1 - deployability/modules/workflow_engine/task.py | 4 ++-- deployability/modules/workflow_engine/workflow_processor.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/deployability/modules/allocation/allocation.py b/deployability/modules/allocation/allocation.py index fb856ea2ce..4b14bc53f1 100755 --- a/deployability/modules/allocation/allocation.py +++ b/deployability/modules/allocation/allocation.py @@ -64,7 +64,7 @@ def __delete(cls, payload: models.DeletionPayload) -> None: Args: payload (DeletionPayload): The payload containing the parameters - for instance deletion. + for instance deletion. """ payload = models.DeletionPayload(**dict(payload)) # Read the data from the track file. diff --git a/deployability/modules/allocation/aws/instance.py b/deployability/modules/allocation/aws/instance.py index 1537c80e05..794baa91f7 100644 --- a/deployability/modules/allocation/aws/instance.py +++ b/deployability/modules/allocation/aws/instance.py @@ -68,9 +68,9 @@ def ssh_connection_info(self) -> ConnectionInfo: ConnectionInfo: SSH connection information. """ return ConnectionInfo(hostname=self._instance.public_dns_name, - user=self._user, - port=22, - private_key=str(self.credentials.key_path)) + user=self._user, + port=22, + private_key=str(self.credentials.key_path)) def __get_credentials(self) -> AWSCredentials: """ diff --git a/deployability/modules/workflow_engine/__main__.py b/deployability/modules/workflow_engine/__main__.py index 8f2dfab0a2..5d9fe0a0c4 100755 --- a/deployability/modules/workflow_engine/__main__.py +++ b/deployability/modules/workflow_engine/__main__.py @@ -1,4 +1,3 @@ - # Copyright (C) 2015, Wazuh Inc. # Created by Wazuh, Inc. . # This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 diff --git a/deployability/modules/workflow_engine/task.py b/deployability/modules/workflow_engine/task.py index 410d4284bf..b9e349a74b 100755 --- a/deployability/modules/workflow_engine/task.py +++ b/deployability/modules/workflow_engine/task.py @@ -50,7 +50,7 @@ def execute(self) -> None: else: task_args.append(f"--{key}={value}") else: - logger.error(f'Coulg not parse arguments {arg}') + logger.error(f'Could not parse arguments {arg}') logger.debug(f'Running task "{self.task_name}" with arguments: {task_args}') @@ -69,7 +69,7 @@ def execute(self) -> None: except subprocess.CalledProcessError as e: error_msg = e.stderr if "KeyboardInterrupt" in error_msg: - raise KeyboardInterrupt(f"Error executing process task with keyboard interrupt") + raise KeyboardInterrupt(f"Error executing process task with keyboard interrupt.") raise Exception(f"Error executing process task {e.stderr}") class DummyTask(Task): diff --git a/deployability/modules/workflow_engine/workflow_processor.py b/deployability/modules/workflow_engine/workflow_processor.py index 6b8dd3d2f7..bd4b19ef7f 100755 --- a/deployability/modules/workflow_engine/workflow_processor.py +++ b/deployability/modules/workflow_engine/workflow_processor.py @@ -35,7 +35,7 @@ def __validate_schema(self, workflow_file: Path | str) -> None: workflow_file (Path | str): Path to the workflow file. """ try: - logger.debug(f"Validating inptu file: {workflow_file}") + logger.debug(f"Validating input file: {workflow_file}") validator = SchemaValidator(self.schema_path, workflow_file) validator.preprocess_data() validator.validateSchema() From 63b120651353e0f12a89b4dc2e438ffc5dab9ffd Mon Sep 17 00:00:00 2001 From: fcaffieri Date: Thu, 15 Feb 2024 12:24:26 -0300 Subject: [PATCH 18/23] Fix review comments --- deployability/modules/allocation/allocation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/allocation/allocation.py b/deployability/modules/allocation/allocation.py index 4b14bc53f1..8f643d6473 100755 --- a/deployability/modules/allocation/allocation.py +++ b/deployability/modules/allocation/allocation.py @@ -64,7 +64,7 @@ def __delete(cls, payload: models.DeletionPayload) -> None: Args: payload (DeletionPayload): The payload containing the parameters - for instance deletion. + for instance deletion. """ payload = models.DeletionPayload(**dict(payload)) # Read the data from the track file. From 8be55697861cb6ec6c45fc8357af4594ef7dff5e Mon Sep 17 00:00:00 2001 From: Antonio Date: Mon, 19 Feb 2024 13:40:29 +0100 Subject: [PATCH 19/23] fix(#4910): Changing Readme and dtt-agents-poc file --- .../modules/workflow_engine/README.md | 28 +++++------ .../examples/dtt1-agents-poc.yaml | 48 +++++++------------ 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/deployability/modules/workflow_engine/README.md b/deployability/modules/workflow_engine/README.md index b513275416..55be493d92 100755 --- a/deployability/modules/workflow_engine/README.md +++ b/deployability/modules/workflow_engine/README.md @@ -53,7 +53,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual manager-os: linux-ubuntu-22.04-amd64 infra-provider: vagrant working-dir: /tmp/dtt1-poc - + tasks: # Generic agent test task - task: "run-agent-tests-{agent}" @@ -63,7 +63,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - testing/main.py + - /modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -78,7 +78,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual foreach: - variable: agents-os as: agent - + # Generic agent test task - task: "run-agent-tests-uninstall-{agent}" description: "Run tests uninstall for the {agent} agent." @@ -87,7 +87,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - testing/main.py + - /modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -101,7 +101,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual foreach: - variable: agents-os as: agent - + # Unique manager allocate task - task: "allocate-manager" description: "Allocate resources for the manager." @@ -110,7 +110,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: large @@ -122,10 +122,10 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: delete - track-output: "{working-dir}/manager-{manager-os}/track.yaml" - + # Generic agent provision task - task: "provision-install-{agent}" description: "Provision resources for the {agent} agent." @@ -134,7 +134,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - provision/main.py + - /modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -147,7 +147,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual foreach: - variable: agents-os as: agent - + # Generic agent provision task - task: "provision-uninstall-{agent}" description: "Provision resources for the {agent} agent." @@ -156,7 +156,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - provision/main.py + - /modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -167,7 +167,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual foreach: - variable: agents-os as: agent - + # Generic agent allocate task - task: "allocate-{agent}" description: "Allocate resources for the {agent} agent." @@ -176,7 +176,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: small @@ -188,7 +188,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - allocation.py + - /modules/allocation/main.py - action: delete - track-output: "{working-dir}/agent-{agent}/track.yaml" foreach: diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index d1b7572acd..0108f15036 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -19,9 +19,9 @@ tasks: with: path: python3 args: - - testing/main.py + - /modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - - dependencies: + - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - agent: "{working-dir}/agent-{agent}/inventory.yaml" - tests: "install,register,stop" @@ -43,7 +43,7 @@ tasks: with: path: python3 args: - - testing/main.py + - /modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -58,22 +58,6 @@ tasks: - variable: agents-os as: agent - # Unique manager provision task - - task: "provision-manager" - description: "Provision the manager." - do: - this: process - with: - path: python3 - args: - - provision/main.py - - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - - install: - - component: wazuh-manager - type: package - depends-on: - - "allocate-manager" - # Unique manager allocate task - task: "allocate-manager" description: "Allocate resources for the manager." @@ -82,7 +66,7 @@ tasks: with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: large @@ -94,7 +78,7 @@ tasks: with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: delete - track-output: "{working-dir}/manager-{manager-os}/track.yaml" @@ -106,7 +90,7 @@ tasks: with: path: python3 args: - - provision/main.py + - /modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -128,7 +112,7 @@ tasks: with: path: python3 args: - - provision/main.py + - /modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -148,21 +132,21 @@ tasks: with: path: python3 args: - - allocation/main.py + - /modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: small - composite-name: "{agent}" - inventory-output: "{working-dir}/agent-{agent}/inventory.yaml" - track-output: "{working-dir}/agent-{agent}/track.yaml" - #cleanup: - # this: process - # with: - # path: python3 - # args: - # - allocation.py - # - action: delete - # - track-output: "{working-dir}/agent-{agent}/track.yaml" + cleanup: + this: process + with: + path: python3 + args: + - /modules/allocation/main.py + - action: delete + - track-output: "{working-dir}/agent-{agent}/track.yaml" foreach: - variable: agents-os as: agent \ No newline at end of file From 88b4f9bc80ee1c793bc4a2418be1783296ee9f09 Mon Sep 17 00:00:00 2001 From: Antonio Date: Mon, 19 Feb 2024 13:55:59 +0100 Subject: [PATCH 20/23] fix(#4910): Fixing Readme and dtt-agents-poc file --- .../modules/workflow_engine/README.md | 34 ++++++++++++++----- .../examples/dtt1-agents-poc.yaml | 34 ++++++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/deployability/modules/workflow_engine/README.md b/deployability/modules/workflow_engine/README.md index 55be493d92..71a7fe6b13 100755 --- a/deployability/modules/workflow_engine/README.md +++ b/deployability/modules/workflow_engine/README.md @@ -63,9 +63,9 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/testing/main.py + - modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - - dependencies: + - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - agent: "{working-dir}/agent-{agent}/inventory.yaml" - tests: "install,register,stop" @@ -87,7 +87,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/testing/main.py + - modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -102,6 +102,22 @@ Initially, Python libraries must be installed. It is recommended to use virtual - variable: agents-os as: agent + # Unique manager provision task + - task: "provision-manager" + description: "Provision the manager." + do: + this: process + with: + path: python3 + args: + - modules/provision/main.py + - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - install: + - component: wazuh-manager + type: package + depends-on: + - "allocate-manager" + # Unique manager allocate task - task: "allocate-manager" description: "Allocate resources for the manager." @@ -110,7 +126,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: large @@ -122,7 +138,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: delete - track-output: "{working-dir}/manager-{manager-os}/track.yaml" @@ -134,7 +150,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/provision/main.py + - modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -156,7 +172,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/provision/main.py + - modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -176,7 +192,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: small @@ -188,7 +204,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: delete - track-output: "{working-dir}/agent-{agent}/track.yaml" foreach: diff --git a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml index 0108f15036..0773ccc4cd 100755 --- a/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml +++ b/deployability/modules/workflow_engine/examples/dtt1-agents-poc.yaml @@ -19,9 +19,9 @@ tasks: with: path: python3 args: - - /modules/testing/main.py + - modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - - dependencies: + - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - agent: "{working-dir}/agent-{agent}/inventory.yaml" - tests: "install,register,stop" @@ -43,7 +43,7 @@ tasks: with: path: python3 args: - - /modules/testing/main.py + - modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" @@ -58,6 +58,22 @@ tasks: - variable: agents-os as: agent + # Unique manager provision task + - task: "provision-manager" + description: "Provision the manager." + do: + this: process + with: + path: python3 + args: + - modules/provision/main.py + - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" + - install: + - component: wazuh-manager + type: package + depends-on: + - "allocate-manager" + # Unique manager allocate task - task: "allocate-manager" description: "Allocate resources for the manager." @@ -66,7 +82,7 @@ tasks: with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: large @@ -78,7 +94,7 @@ tasks: with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: delete - track-output: "{working-dir}/manager-{manager-os}/track.yaml" @@ -90,7 +106,7 @@ tasks: with: path: python3 args: - - /modules/provision/main.py + - modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - install: @@ -112,7 +128,7 @@ tasks: with: path: python3 args: - - /modules/provision/main.py + - modules/provision/main.py - inventory-agent: "{working-dir}/agent-{agent}/inventory.yaml" - inventory-manager: "{working-dir}/manager-{manager-os}/inventory.yaml" - uninstall: @@ -132,7 +148,7 @@ tasks: with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: create - provider: "{infra-provider}" - size: small @@ -144,7 +160,7 @@ tasks: with: path: python3 args: - - /modules/allocation/main.py + - modules/allocation/main.py - action: delete - track-output: "{working-dir}/agent-{agent}/track.yaml" foreach: From a405f8396972422b59508e2cb6dee592a95b0a50 Mon Sep 17 00:00:00 2001 From: Antonio Kim <125690423+pro-akim@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:38:24 +0100 Subject: [PATCH 21/23] fix(#4910): Changing Tecnical documentation path --- deployability/modules/workflow_engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/workflow_engine/README.md b/deployability/modules/workflow_engine/README.md index 71a7fe6b13..2bcfe94b09 100755 --- a/deployability/modules/workflow_engine/README.md +++ b/deployability/modules/workflow_engine/README.md @@ -284,7 +284,7 @@ tasks: with: path: python3 args: - - testing/main.py + - modules/testing/main.py - inventory: "{working-dir}/agent-{agent}/inventory.yaml" - dependencies: - manager: "{working-dir}/manager-{manager-os}/inventory.yaml" From a4577984a36bb38601512e290611e4025b631da4 Mon Sep 17 00:00:00 2001 From: quebim Date: Mon, 19 Feb 2024 11:16:04 -0300 Subject: [PATCH 22/23] Remove duplicated examples --- .../multiple_linear_independant_flow.yaml | 69 ------------------- ...e_linear_independant_flow_templetized.yaml | 55 --------------- .../examples/single_linear_flow.yaml | 38 ---------- 3 files changed, 162 deletions(-) delete mode 100644 deployability/workflow_engine/examples/multiple_linear_independant_flow.yaml delete mode 100644 deployability/workflow_engine/examples/multiple_linear_independant_flow_templetized.yaml delete mode 100644 deployability/workflow_engine/examples/single_linear_flow.yaml diff --git a/deployability/workflow_engine/examples/multiple_linear_independant_flow.yaml b/deployability/workflow_engine/examples/multiple_linear_independant_flow.yaml deleted file mode 100644 index 7f0ae53560..0000000000 --- a/deployability/workflow_engine/examples/multiple_linear_independant_flow.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2015, Wazuh Inc. -# Created by Wazuh, Inc. . -# This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 -version: 0.1 -description: This is a basic example of two linear and independant flows. - -tasks: - - task: "A1" - description: "Task A1" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task A1" - depends-on: - - "B1" - - task: "B1" - description: "Task B1" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task B1" - depends-on: - - "C1" - - task: "C1" - description: "Task C1" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task C1" - - task: "A2" - description: "Task A2" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task A2" - depends-on: - - "B2" - - task: "B2" - description: "Task B2" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task B2" - depends-on: - - "C2" - - task: "C2" - description: "Task C2" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task C2" diff --git a/deployability/workflow_engine/examples/multiple_linear_independant_flow_templetized.yaml b/deployability/workflow_engine/examples/multiple_linear_independant_flow_templetized.yaml deleted file mode 100644 index b4c55c96d7..0000000000 --- a/deployability/workflow_engine/examples/multiple_linear_independant_flow_templetized.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2015, Wazuh Inc. -# Created by Wazuh, Inc. . -# This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 -version: 0.1 -description: This is a basic example of two linear and independant flows using templates. - -variables: - index: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 -tasks: - - task: "A{i}" - description: "Task A{i}" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task A{i}" - depends-on: - - "B{i}" - foreach: - - variable: index - as: i - - task: "B{i}" - description: "Task B{i}" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task B{i}" - depends-on: - - "C{i}" - foreach: - - variable: index - as: i - - task: "C{i}" - description: "Task C{i}" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task C{i}" - foreach: - - variable: index - as: i diff --git a/deployability/workflow_engine/examples/single_linear_flow.yaml b/deployability/workflow_engine/examples/single_linear_flow.yaml deleted file mode 100644 index 25c6965a00..0000000000 --- a/deployability/workflow_engine/examples/single_linear_flow.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2015, Wazuh Inc. -# Created by Wazuh, Inc. . -# This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 -version: 0.1 -description: This is a basic example of linear flow. - -tasks: - - task: "A" - description: "Task A" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task A" - depends-on: - - "B" - - task: "B" - description: "Task B" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task B" - depends-on: - - "C" - - task: "C" - description: "Task C" - do: - this: process - with: - path: /bin/echo - args: - - -n - - "Running task C" From e64999372b505c15ebd95872fa91de3af3f54ac5 Mon Sep 17 00:00:00 2001 From: Kevin Ledesma Date: Mon, 19 Feb 2024 12:34:14 -0300 Subject: [PATCH 23/23] Update README.md --- deployability/modules/workflow_engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/workflow_engine/README.md b/deployability/modules/workflow_engine/README.md index 2bcfe94b09..51eaad4ad6 100755 --- a/deployability/modules/workflow_engine/README.md +++ b/deployability/modules/workflow_engine/README.md @@ -24,7 +24,7 @@ Initially, Python libraries must be installed. It is recommended to use virtual 3. Install requirements: ```bash - pip3 install -r /deployability/deps/requirements.txt + pip3 install -r deployability/deps/requirements.txt ``` 4. Install the Workflow engine library and its launcher: