Add ECS mappings generator, documentation and files for vulnerability…

… detector

ecs/.gitignore

@@ -0,0 +1 @@
+**/mappings

ecs/README.md

@@ -0,0 +1,73 @@
+### ECS mappings generator
+
+This script generates the ECS mappings for the Wazuh indices.
+
+#### Requirements
+
+- ECS repository clone. The script is meant to be launched from the root level of that repository.
+- Python 3.6 or higher
+- jq
+
+#### Folder structrue
+
+There is a folder for each module. Inside each folder, there is a `fields` folder with the required 
+files to generate the mappings. These are the inputs for the ECS generator.
+
+#### Usage
+
+**Copy the `generate.sh` script to the root level of the ECS repository.**
+
+Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments, 
+plus 2 optional arguments to upload the mappings to the Wazuh indexer (using **composable** indexes).
+
+
+```plaintext
+Usage: ./generate.sh <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]
+  * ECS_VERSION: ECS version to generate mappings for
+  * INDEXER_SRC: Path to the wazuh-indexer repository
+  * MODULE: Module to generate mappings for
+  * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200
+Example: ./generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200
+``` 
+
+For example, to generate the mappings for the `vulnerability-detector` module using the
+ECS version `v8.10.0` and the Wazuh indexer in path `~/wazuh/wazuh-indexer`:
+```bash
+./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector
+```
+
+#### Output
+
+A new `mappings` folder will be created inside the module folder, containing all the generated files.
+The files are versioned using the ECS version, so different versions of the same module can be generated.
+For our use case, the most important files are under `mappings/<ECS_VERSION>/generated/elasticsearch/legacy/`:
+
+- `template.json`: Elasticsearch compatible index template for the module
+- `opensearch-template.json`: OpenSearch compatible index template for the module
+
+The original output is `template.json`, which is not compatible with OpenSearch by default. In order 
+to make this template compatible with OpenSearch, the following changes are made:
+
+- the `order` property is renamed to `priority`. 
+- the `mappings` and `settings` properties are nested under the `template` property.
+
+The script takes care of these changes automatically, generating the `opensearch-template.json` file as a result.
+
+#### Adding new mappings
+
+The easiest way to create mappings for a new module is to take a previous one as a base.
+Copy a folder and rename it to the new module name. Then, edit the `fields` files to
+match the new module fields.
+
+The name of the folder will be the name of the module to be passed to the script. All 3 files
+are required.
+
+- `fields/subset.yml`: This file contains the subset of ECS fields to be used for the module.
+- `fields/template-settings-legacy.json`: This file contains the legacy template settings for the module.
+- `fields/template-settings.json`: This file contains the composable template settings for the module.
+
+#### References
+
+- [ECS repository](https://github.com/elastic/ecs)
+- [ECS usage](https://github.com/elastic/ecs/blob/main/USAGE.md)
+- [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html)

ecs/generate.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Function to display usage information
+show_usage() {
+  echo "Usage: $0 <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]"
+  echo "  * ECS_VERSION: ECS version to generate mappings for"
+  echo "  * INDEXER_SRC: Path to the wazuh-indexer repository"
+  echo "  * MODULE: Module to generate mappings for"
+  echo "  * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200"
+  echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
+}
+
+# Function to generate mappings
+generate_mappings() {
+  ECS_VERSION="$1"
+  INDEXER_SRC="$2"
+  MODULE="$3"
+  UPLOAD="$4"
+  URL="$5"
+
+  IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
+  OUT_DIR="$INDEXER_SRC/ecs/$MODULE/mappings/$ECS_VERSION"
+
+  # Ensure the output directory exists
+  mkdir -p "$OUT_DIR" || exit 1
+
+  # Generate mappings
+  python scripts/generator.py --strict --ref "$ECS_VERSION" \
+    --subset "$IN_FILES_DIR/subset.yml" \
+    --template-settings "$IN_FILES_DIR/template-settings.json" \
+    --template-settings-legacy "$IN_FILES_DIR/template-settings-legacy.json" \
+    --out "$OUT_DIR" || exit 1
+
+  # Replace "match_only_text" type (not supported by OpenSearch) with "text"
+  echo "Replacing \"match_only_text\" type with \"text\""
+  find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;
+
+  # Transform legacy index template for OpenSearch compatibility
+  cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
+    "index_patterns": .index_patterns,
+    "priority": .order,
+    "template": {
+      "settings": .settings,
+      "mappings": .mappings
+    }
+  }' > "$OUT_DIR/generated/elasticsearch/legacy/opensearch-template.json"
+
+  # Check if the --upload flag has been provided
+  if [ "$UPLOAD" == "--upload" ]; then
+    upload_mappings "$OUT_DIR" "$URL" || exit 1
+  fi
+
+  echo "Mappings saved to $OUT_DIR"
+}
+
+# Function to upload generated composable index template to the OpenSearch cluster
+upload_mappings() {
+  OUT_DIR="$1"
+  URL="$2"
+
+  echo "Uploading index template to the OpenSearch cluster"
+  for file in "$OUT_DIR/generated/elasticsearch/composable/component"/*.json; do
+    component_name=$(basename "$file" .json)
+    echo "Uploading $component_name"
+    curl -u admin:admin -X PUT "$URL/_component_template/$component_name?pretty" -H 'Content-Type: application/json' -d@"$file" || exit 1
+  done
+}
+
+# Check if the minimum required arguments have been provided
+if [ $# -lt 3 ]; then
+  show_usage
+  exit 1
+fi
+
+# Parse command line arguments
+ECS_VERSION="$1"
+INDEXER_SRC="$2"
+MODULE="$3"
+UPLOAD="${4:-false}"
+URL="${5:-https://localhost:9200}"
+
+# Generate mappings
+generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL"

ecs/vulnerability-detector/fields/subset.yml

@@ -0,0 +1,19 @@
+---
+name: vulnerability_detector
+fields:
+  base:
+    fields: "*"
+  agent:
+    fields: "*"
+  ecs:
+    fields: "*"
+  event:
+    fields: "*"
+  package:
+    fields: "*"
+  host:
+    fields: 
+      os:
+        fields: "*"
+  vulnerability:
+    fields: "*"

ecs/vulnerability-detector/fields/template-settings-legacy.json

@@ -0,0 +1,16 @@
+{
+  "index_patterns": ["wazuh-states-vulnerabilities"],
+  "order": 1,
+  "settings": {
+    "index": {
+      "codec" : "best_compression",
+      "mapping": {
+        "total_fields": {
+          "limit": 1000
+        }
+      },
+      "refresh_interval": "2s"
+    }
+  }
+}
+

ecs/vulnerability-detector/fields/template-settings.json

@@ -0,0 +1,16 @@
+{
+  "index_patterns": ["wazuh-states-vulnerabilities"],
+  "priority": 1,
+  "template": {
+    "settings": {
+      "index": {
+        "codec": "best_compression",
+        "mapping": {
+          "total_fields": {
+            "limit": 2000
+          }
+        }
+      }
+    }
+  }
+}