Merge pull request #4841 from wazuh/enhancement/4751-dtt1-iteration-2…

…-poc

Enhancement/4751 dtt1 iteration 2 poc

.gitignore

@@ -5,6 +5,7 @@ __pycache__
 venv
 wazuh_testing.egg-info
 dist
+.env
 
 # Python bytecode files
 *.pyc
@@ -44,6 +45,7 @@ Temporary Items
 
 ### VisualStudioCode ###
 .vscode/*
+.vscode/
 .mypy_cache/
 .vs/*
 
@@ -67,3 +69,6 @@ ansible.log
 
 # Ssh keys
 key
+
+#Python
+__pycache__/

deployability/Jenkinsfiles/Launcher.groovy

@@ -0,0 +1,31 @@
+
+String jenkins_reference = params.getOrDefault('JENKINS_REFERENCE', 'enhancement/4751-dtt1-iteration-2-poc')
+String launcher_path = "launchers"
+String task_flow_launcher = "provision.py"
+String workflow = "modules/workflow_engine/examples/dtt1-managers.yaml"
+String schema = "modules/workflow_engine/schema.json"
+
+// Jenkinsfile
+
+node {
+
+  try {
+    stage('Clone Repo') {
+      print("Clone repository")
+      git branch: "${JENKINS_REFERENCE}", url: 'https://github.com/wazuh/wazuh-qa.git'
+    }
+
+    stage('Launch Task Flow') {
+      print("Launch Task Flow dry run")
+      sh "cd ${env.WORKSPACE}/deployability && python3 ${launcher_path}/${task_flow_launcher} ${workflow} --dry-run"
+
+      print("Launch Task Flow")
+      sh "cd ${env.WORKSPACE}/deployability && python3 ${launcher_path}/${task_flow_launcher} ${workflow} ${schema}"
+    }
+  }
+  finally{
+    stage('Remove venv') {
+      sh "rm -rf ${env.WORKSPACE}/deployability/venv"
+    }
+  }
+}

deployability/deps/remote_requirements.txt

@@ -0,0 +1,5 @@
+# python3-pip
+pytest>=7.4.2,<8.0.0
+chardet==5.2.0
+chardet==5.2.0
+pytest-tinybird==0.2.0

deployability/deps/requirements.txt

@@ -0,0 +1,10 @@
+ansible_runner==2.3.4
+boto3==1.29.1
+pydantic==2.5.2
+ansible
+ruamel.yaml==0.18.5
+ruamel.yaml.clib==0.2.8
+graphlib==0.9.5
+jsonschema==3.2.0
+PyYAML==6.0.1
+colorlog==6.8.0

deployability/launchers/allocation.py

@@ -0,0 +1,29 @@
+import argparse
+import os
+import sys
+
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(project_root)
+
+from modules.allocation import Allocator
+from modules.allocation.generic.models import InputPayload
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Infrastructure providing tool")
+    parser.add_argument("--provider", choices=['vagrant', 'aws', None], required=False,  default=None)
+    parser.add_argument("--size", choices=['micro', 'small', 'medium', 'large', None], required=False, default=None)
+    parser.add_argument("--composite-name", required=False, default=None)
+    parser.add_argument("--action", choices=['create', 'delete'], required=False, default='create')
+    parser.add_argument("--custom-credentials", required=False, default=None)
+    parser.add_argument("--track-output", required=False, default='/tmp/wazuh-qa/track.yml')
+    parser.add_argument("--inventory-output", required=False, default='/tmp/wazuh-qa/inventory.yml')
+    parser.add_argument("--working-dir", required=False, default='/tmp/wazuh-qa')
+    return parser.parse_args()
+
+
+def main():
+    Allocator.run(InputPayload(**vars(parse_arguments())))
+
+
+if __name__ == "__main__":
+    main()

deployability/launchers/provision.py

@@ -0,0 +1,29 @@
+import argparse
+import os
+import sys
+import ast
+import json
+
+# ---------------- Vars ------------------------
+
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(project_root)
+
+from modules.provision import Provision, models
+
+# ---------------- Methods ---------------------
+
+
+def parse_arguments():
+  parser = argparse.ArgumentParser(description="Provision infraestructure tool")
+  parser.add_argument("--inventory-agent", default=None, help="Inventory with agent host information")
+  parser.add_argument("--inventory-manager", default=None, help="Inventory with manager host information")
+  parser.add_argument('--install',  action='append', default=[], help='List of dictionaries for installation.')
+  parser.add_argument("--custom-credentials", required=False, default=None)
+  return parser.parse_args()
+
+if __name__ == "__main__":
+  provision = Provision(models.InputPayload(**vars(parse_arguments())))
+  provision.run()
+
+# ----------------------------------------------

deployability/launchers/test.py

@@ -0,0 +1,35 @@
+import argparse
+import sys
+import os
+
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(project_root)
+
+from modules.testing import Tester, InputPayload
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Wazuh testing tool")
+    parser.add_argument("--inventory", required=True)
+    parser.add_argument("--tests", required=True)
+    parser.add_argument("--component", choices=['manager', 'agent'], required=True)
+    parser.add_argument("--dependency", required=False)
+    parser.add_argument("--cleanup", required=False, default=True)
+    parser.add_argument("--wazuh-version", required=True)
+    parser.add_argument("--wazuh-revision", required=True)
+    parser.add_argument("--wazuh-branch", required=False)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    Tester.run(InputPayload(**vars(parse_arguments())))
+
+
+
+# linux-ubuntu-20.04-amd64:                                   
+#   hosts:
+#     VAGRANT-F6FD6643-B41E-4112-A652-3CFF8CC26F51:
+#       ansible_host: 127.0.0.1
+#       ansible_port: 2222
+#       ansible_ssh_private_key_file: C:\tmp\wazuh-qa\VAGRANT-F6FD6643-B41E-4112-A652-3CFF8CC26F51\instance_key        
+#       ansible_user: vagrant

deployability/launchers/workflow_engine.py

@@ -0,0 +1,49 @@
+# Copyright (C) 2015, Wazuh Inc.
+# Created by Wazuh, Inc. <[email protected]>.
+# This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2
+
+import os
+import sys
+import argparse
+import logging
+import colorlog
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(project_root)
+from modules.workflow_engine.workflow_processor import WorkflowProcessor
+from modules.generic import SchemaValidator
+
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(description='Execute tasks in a workflow.')
+    parser.add_argument('workflow_file', type=str, help='Path to the workflow file (YAML format).')
+    parser.add_argument('schema_file', type=str, default="./schema.json", help='Path to the schema definition file.')
+    parser.add_argument('--threads', type=int, default=1, help='Number of threads to use for parallel execution.')
+    parser.add_argument('--dry-run', action='store_true', help='Display the plan without executing tasks.')
+    parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO',
+                        help='Log level.')
+    return parser.parse_args()
+
+def setup_logger(log_level: str) -> None:
+    """Setup logger."""
+    logger = logging.getLogger()
+    console_handler = colorlog.StreamHandler()
+    console_handler.setFormatter(colorlog.ColoredFormatter("%(log_color)s[%(asctime)s] [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(console_handler)
+    logger.setLevel(log_level)
+
+def main() -> None:
+    """Main entry point."""
+
+    args = parse_arguments()
+    validator = SchemaValidator(args.schema_file, args.workflow_file)
+    validator.preprocess_data()
+    validator.validateSchema()
+    setup_logger(args.log_level)
+    processor = WorkflowProcessor(args.workflow_file, args.dry_run, args.threads)
+    processor.run()
+
+
+if __name__ == "__main__":
+    main()

deployability/modules/__init__.py

@@ -0,0 +1,5 @@
+from .provision import Provision
+from .generic import Ansible
+from .allocation.vagrant import VagrantProvider
+from .allocation.aws import AWSProvider
+from .generic import SchemaValidator

deployability/modules/allocation/__init__.py

@@ -0,0 +1 @@
+from .allocation import Allocator

deployability/modules/allocation/allocation.py

@@ -0,0 +1,113 @@
+import yaml
+
+from pathlib import Path
+
+from .generic import Instance, Provider, models
+from .aws.provider import AWSProvider
+from .vagrant.provider import VagrantProvider
+
+PROVIDERS = {'vagrant': VagrantProvider, 'aws': AWSProvider}
+
+
+class Allocator:
+    """
+    Allocator class to manage instances based on the payload action.
+    """
+    @classmethod
+    def run(cls, payload: models.InputPayload) -> None:
+        """
+        Executes the appropriate method based on the payload action.
+
+        Args:
+            payload (InputPayload): The payload containing the action parameters.
+        """
+        payload = models.InputPayload(**dict(payload))
+        # Detect the action and call the appropriate method.
+        if payload.action == 'create':
+            print(f"Creating instance at {payload.working_dir}")
+            return cls.__create(payload)
+        elif payload.action == 'delete':
+            print(f"Deleting instance from trackfile {payload.track_output}")
+            return cls.__delete(payload)
+
+    # Internal methods
+
+    @classmethod
+    def __create(cls, payload: models.CreationPayload):
+        """
+        Creates an instance and generates the inventory and track files.
+
+        Args:
+            payload (CreationPayload): The payload containing the parameters
+                                              for instance creation.
+        """
+        instance_params = models.CreationPayload(**dict(payload))
+        provider: Provider = PROVIDERS[payload.provider]()
+        instance = provider.create_instance(
+            payload.working_dir, instance_params)
+        print(f"Instance {instance.identifier} created.")
+        # Start the instance
+        instance.start()
+        print(f"Instance {instance.identifier} started.")
+        # Generate the inventory and track files.
+        cls.__generate_inventory(instance, payload.inventory_output)
+        cls.__generate_track_file(instance, payload.provider, payload.track_output)
+
+    @classmethod
+    def __delete(cls, payload: models.DeletionPayload) -> None:
+        """
+        Deletes an instance based on the data from the track file.
+
+        Args:
+            payload (DeletionPayload): The payload containing the parameters
+                                              for instance deletion.
+        """
+        payload = models.DeletionPayload(**dict(payload))
+        # Read the data from the track file.
+        with open(payload.track_output, 'r') as f:
+            track = models.TrackOutput(**yaml.safe_load(f))
+        provider = PROVIDERS[track.provider]()
+        provider.destroy_instance(track.instance_dir, track.identifier)
+        print(f"Instance {track.identifier} deleted.")
+
+    @staticmethod
+    def __generate_inventory(instance: Instance, inventory_path: Path) -> None:
+        """
+        Generates an inventory file.
+
+        Args:
+            instance (Instance): The instance for which the inventory file is generated.
+            inventory_path (Path): The path where the inventory file will be generated.
+        """
+        inventory_path = Path(inventory_path)
+        if not inventory_path.parent.exists():
+            inventory_path.parent.mkdir(parents=True, exist_ok=True)
+        ssh_config = instance.ssh_connection_info()
+        inventory = models.InventoryOutput(ansible_host=ssh_config.hostname,
+                                           ansible_user=ssh_config.user,
+                                           ansible_port=ssh_config.port,
+                                           ansible_ssh_private_key_file=str(ssh_config.private_key))
+        with open(inventory_path, 'w') as f:
+            yaml.dump(inventory.model_dump(), f)
+        print(f"\nInventory file generated at {inventory_path}")
+
+    @staticmethod
+    def __generate_track_file(instance: Instance, provider_name: str,  track_path: Path) -> None:
+        """
+        Generates a track file.
+
+        Args:
+            instance (Instance): The instance for which the track file is to be generated.
+            provider_name (str): The name of the provider.
+            track_path (Path): The path where the track file will be generated.
+        """
+        track_path = Path(track_path)
+        if not track_path.parent.exists():
+            track_path.parent.mkdir(parents=True, exist_ok=True)
+        track = models.TrackOutput(identifier=instance.identifier,
+                                   provider=provider_name,
+                                   instance_dir=str(instance.path),
+                                   key_path=str(instance.credentials.key_path))
+        with open(track_path, 'w') as f:
+            yaml.dump(track.model_dump(), f)
+        print(f"\nTrack file generated at {track_path}")

deployability/modules/allocation/aws/__init__.py

@@ -0,0 +1,3 @@
+from .credentials import AWSCredentials
+from .instance import AWSInstance
+from .provider import AWSProvider