Add ECS mappings generator (#36)

* Add ECS mappings generator, documentation and files for vulnerability detector

* Add event generator script

* Update template settings

---------

Signed-off-by: Álex Ruiz <[email protected]>

ecs/.gitignore

@@ -0,0 +1,3 @@
+**/mappings
+*.log
+generatedData.json

ecs/README.md

@@ -0,0 +1,104 @@
+## ECS mappings generator
+
+This script generates the ECS mappings for the Wazuh indices.
+
+### Requirements
+
+- ECS repository clone. The script is meant to be launched from the root level of that repository.
+- Python 3.6 or higher
+- jq
+
+### Folder structure
+
+There is a folder for each module. Inside each folder, there is a `fields` folder with the required
+files to generate the mappings. These are the inputs for the ECS generator.
+
+### Usage
+
+**Copy the `generate.sh` script to the root level of the ECS repository.**
+
+Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments,
+plus 2 optional arguments to upload the mappings to the Wazuh indexer (using **composable** indexes).
+
+```plaintext
+Usage: ./generate.sh <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]
+  * ECS_VERSION: ECS version to generate mappings for
+  * INDEXER_SRC: Path to the wazuh-indexer repository
+  * MODULE: Module to generate mappings for
+  * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200
+Example: ./generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200
+```
+
+For example, to generate the mappings for the `vulnerability-detector` module using the
+ECS version `v8.10.0` and the Wazuh indexer in path `~/wazuh/wazuh-indexer`:
+
+```bash
+./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector
+```
+
+### Output
+
+A new `mappings` folder will be created inside the module folder, containing all the generated files.
+The files are versioned using the ECS version, so different versions of the same module can be generated.
+For our use case, the most important files are under `mappings/<ECS_VERSION>/generated/elasticsearch/legacy/`:
+
+- `template.json`: Elasticsearch compatible index template for the module
+- `opensearch-template.json`: OpenSearch compatible index template for the module
+
+The original output is `template.json`, which is not compatible with OpenSearch by default. In order
+to make this template compatible with OpenSearch, the following changes are made:
+
+- the `order` property is renamed to `priority`.
+- the `mappings` and `settings` properties are nested under the `template` property.
+
+The script takes care of these changes automatically, generating the `opensearch-template.json` file as a result.
+
+### Upload
+
+You can either upload the index template using cURL or the UI (dev tools).
+
+```bash
+curl -u admin:admin -k -X PUT "https://indexer:9200/_index_template/wazuh-vulnerability-detector" -H "Content-Type: application/json" -d @opensearch-template.json
+```
+
+Notes:
+- PUT and POST are interchangeable.
+- The name of the index template does not matter. Any name can be used.
+- Adjust credentials and URL accordingly.
+
+### Adding new mappings
+
+The easiest way to create mappings for a new module is to take a previous one as a base.
+Copy a folder and rename it to the new module name. Then, edit the `fields` files to
+match the new module fields.
+
+The name of the folder will be the name of the module to be passed to the script. All 3 files
+are required.
+
+- `fields/subset.yml`: This file contains the subset of ECS fields to be used for the module.
+- `fields/template-settings-legacy.json`: This file contains the legacy template settings for the module.
+- `fields/template-settings.json`: This file contains the composable template settings for the module.
+
+### Event generator
+
+For testing purposes, the script `generate_events.py` can be used to generate events for a given module.
+Currently, it is only able to generate events for the `vulnerability-detector` module. To support other
+modules, please extend or refactor the script.
+
+The script prompts for the required parameters, so it can be launched without arguments:
+
+```bash
+./event_generator.py
+```
+
+The script will generate a JSON file with the events, and will also ask whether to upload them to the
+indexer. If the upload option is selected, the script will ask for the indexer URL and port, credentials,
+and index name.
+
+The script uses a log file. Check it out for debugging or additional information.
+
+#### References
+
+- [ECS repository](https://github.com/elastic/ecs)
+- [ECS usage](https://github.com/elastic/ecs/blob/main/USAGE.md)
+- [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html)

ecs/generate.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Function to display usage information
+show_usage() {
+  echo "Usage: $0 <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]"
+  echo "  * ECS_VERSION: ECS version to generate mappings for"
+  echo "  * INDEXER_SRC: Path to the wazuh-indexer repository"
+  echo "  * MODULE: Module to generate mappings for"
+  echo "  * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200"
+  echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
+}
+
+# Function to generate mappings
+generate_mappings() {
+  ECS_VERSION="$1"
+  INDEXER_SRC="$2"
+  MODULE="$3"
+  UPLOAD="$4"
+  URL="$5"
+
+  IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
+  OUT_DIR="$INDEXER_SRC/ecs/$MODULE/mappings/$ECS_VERSION"
+
+  # Ensure the output directory exists
+  mkdir -p "$OUT_DIR" || exit 1
+
+  # Generate mappings
+  python scripts/generator.py --strict --ref "$ECS_VERSION" \
+    --subset "$IN_FILES_DIR/subset.yml" \
+    --template-settings "$IN_FILES_DIR/template-settings.json" \
+    --template-settings-legacy "$IN_FILES_DIR/template-settings-legacy.json" \
+    --out "$OUT_DIR" || exit 1
+
+  # Replace "match_only_text" type (not supported by OpenSearch) with "text"
+  echo "Replacing \"match_only_text\" type with \"text\""
+  find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;
+
+  # Transform legacy index template for OpenSearch compatibility
+  cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
+    "index_patterns": .index_patterns,
+    "priority": .order,
+    "template": {
+      "settings": .settings,
+      "mappings": .mappings
+    }
+  }' >"$OUT_DIR/generated/elasticsearch/legacy/opensearch-template.json"
+
+  # Check if the --upload flag has been provided
+  if [ "$UPLOAD" == "--upload" ]; then
+    upload_mappings "$OUT_DIR" "$URL" || exit 1
+  fi
+
+  echo "Mappings saved to $OUT_DIR"
+}
+
+# Function to upload generated composable index template to the OpenSearch cluster
+upload_mappings() {
+  OUT_DIR="$1"
+  URL="$2"
+
+  echo "Uploading index template to the OpenSearch cluster"
+  for file in "$OUT_DIR/generated/elasticsearch/composable/component"/*.json; do
+    component_name=$(basename "$file" .json)
+    echo "Uploading $component_name"
+    curl -u admin:admin -X PUT "$URL/_component_template/$component_name?pretty" -H 'Content-Type: application/json' -d@"$file" || exit 1
+  done
+}
+
+# Check if the minimum required arguments have been provided
+if [ $# -lt 3 ]; then
+  show_usage
+  exit 1
+fi
+
+# Parse command line arguments
+ECS_VERSION="$1"
+INDEXER_SRC="$2"
+MODULE="$3"
+UPLOAD="${4:-false}"
+URL="${5:-https://localhost:9200}"
+
+# Generate mappings
+generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL"