diff --git a/ecs/.gitignore b/ecs/.gitignore new file mode 100644 index 0000000000000..c3e3dcdda5d21 --- /dev/null +++ b/ecs/.gitignore @@ -0,0 +1 @@ +**/mappings \ No newline at end of file diff --git a/ecs/README.md b/ecs/README.md new file mode 100644 index 0000000000000..eb74130e74caa --- /dev/null +++ b/ecs/README.md @@ -0,0 +1,73 @@ +### ECS mappings generator + +This script generates the ECS mappings for the Wazuh indices. + +#### Requirements + +- ECS repository clone. The script is meant to be launched from the root level of that repository. +- Python 3.6 or higher +- jq + +#### Folder structrue + +There is a folder for each module. Inside each folder, there is a `fields` folder with the required +files to generate the mappings. These are the inputs for the ECS generator. + +#### Usage + +**Copy the `generate.sh` script to the root level of the ECS repository.** + +Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments, +plus 2 optional arguments to upload the mappings to the Wazuh indexer (using **composable** indexes). + + +```plaintext +Usage: ./generate.sh [--upload ] + * ECS_VERSION: ECS version to generate mappings for + * INDEXER_SRC: Path to the wazuh-indexer repository + * MODULE: Module to generate mappings for + * --upload : Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200 +Example: ./generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200 +``` + +For example, to generate the mappings for the `vulnerability-detector` module using the +ECS version `v8.10.0` and the Wazuh indexer in path `~/wazuh/wazuh-indexer`: +```bash +./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector +``` + +#### Output + +A new `mappings` folder will be created inside the module folder, containing all the generated files. +The files are versioned using the ECS version, so different versions of the same module can be generated. +For our use case, the most important files are under `mappings//generated/elasticsearch/legacy/`: + +- `template.json`: Elasticsearch compatible index template for the module +- `opensearch-template.json`: OpenSearch compatible index template for the module + +The original output is `template.json`, which is not compatible with OpenSearch by default. In order +to make this template compatible with OpenSearch, the following changes are made: + +- the `order` property is renamed to `priority`. +- the `mappings` and `settings` properties are nested under the `template` property. + +The script takes care of these changes automatically, generating the `opensearch-template.json` file as a result. + +#### Adding new mappings + +The easiest way to create mappings for a new module is to take a previous one as a base. +Copy a folder and rename it to the new module name. Then, edit the `fields` files to +match the new module fields. + +The name of the folder will be the name of the module to be passed to the script. All 3 files +are required. + +- `fields/subset.yml`: This file contains the subset of ECS fields to be used for the module. +- `fields/template-settings-legacy.json`: This file contains the legacy template settings for the module. +- `fields/template-settings.json`: This file contains the composable template settings for the module. + +#### References + +- [ECS repository](https://github.com/elastic/ecs) +- [ECS usage](https://github.com/elastic/ecs/blob/main/USAGE.md) +- [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) diff --git a/ecs/generate.sh b/ecs/generate.sh new file mode 100755 index 0000000000000..9cf44f53ea027 --- /dev/null +++ b/ecs/generate.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Function to display usage information +show_usage() { + echo "Usage: $0 [--upload ]" + echo " * ECS_VERSION: ECS version to generate mappings for" + echo " * INDEXER_SRC: Path to the wazuh-indexer repository" + echo " * MODULE: Module to generate mappings for" + echo " * --upload : Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200" + echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200" +} + +# Function to generate mappings +generate_mappings() { + ECS_VERSION="$1" + INDEXER_SRC="$2" + MODULE="$3" + UPLOAD="$4" + URL="$5" + + IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields" + OUT_DIR="$INDEXER_SRC/ecs/$MODULE/mappings/$ECS_VERSION" + + # Ensure the output directory exists + mkdir -p "$OUT_DIR" || exit 1 + + # Generate mappings + python scripts/generator.py --strict --ref "$ECS_VERSION" \ + --subset "$IN_FILES_DIR/subset.yml" \ + --template-settings "$IN_FILES_DIR/template-settings.json" \ + --template-settings-legacy "$IN_FILES_DIR/template-settings-legacy.json" \ + --out "$OUT_DIR" || exit 1 + + # Replace "match_only_text" type (not supported by OpenSearch) with "text" + echo "Replacing \"match_only_text\" type with \"text\"" + find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \; + + # Transform legacy index template for OpenSearch compatibility + cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{ + "index_patterns": .index_patterns, + "priority": .order, + "template": { + "settings": .settings, + "mappings": .mappings + } + }' > "$OUT_DIR/generated/elasticsearch/legacy/opensearch-template.json" + + # Check if the --upload flag has been provided + if [ "$UPLOAD" == "--upload" ]; then + upload_mappings "$OUT_DIR" "$URL" || exit 1 + fi + + echo "Mappings saved to $OUT_DIR" +} + +# Function to upload generated composable index template to the OpenSearch cluster +upload_mappings() { + OUT_DIR="$1" + URL="$2" + + echo "Uploading index template to the OpenSearch cluster" + for file in "$OUT_DIR/generated/elasticsearch/composable/component"/*.json; do + component_name=$(basename "$file" .json) + echo "Uploading $component_name" + curl -u admin:admin -X PUT "$URL/_component_template/$component_name?pretty" -H 'Content-Type: application/json' -d@"$file" || exit 1 + done +} + +# Check if the minimum required arguments have been provided +if [ $# -lt 3 ]; then + show_usage + exit 1 +fi + +# Parse command line arguments +ECS_VERSION="$1" +INDEXER_SRC="$2" +MODULE="$3" +UPLOAD="${4:-false}" +URL="${5:-https://localhost:9200}" + +# Generate mappings +generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" diff --git a/ecs/vulnerability-detector/fields/subset.yml b/ecs/vulnerability-detector/fields/subset.yml new file mode 100644 index 0000000000000..8ef4bde40e2f8 --- /dev/null +++ b/ecs/vulnerability-detector/fields/subset.yml @@ -0,0 +1,19 @@ +--- +name: vulnerability_detector +fields: + base: + fields: "*" + agent: + fields: "*" + ecs: + fields: "*" + event: + fields: "*" + package: + fields: "*" + host: + fields: + os: + fields: "*" + vulnerability: + fields: "*" \ No newline at end of file diff --git a/ecs/vulnerability-detector/fields/template-settings-legacy.json b/ecs/vulnerability-detector/fields/template-settings-legacy.json new file mode 100644 index 0000000000000..51d9a6e80b4ad --- /dev/null +++ b/ecs/vulnerability-detector/fields/template-settings-legacy.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["wazuh-states-vulnerabilities"], + "order": 1, + "settings": { + "index": { + "codec" : "best_compression", + "mapping": { + "total_fields": { + "limit": 1000 + } + }, + "refresh_interval": "2s" + } + } +} + diff --git a/ecs/vulnerability-detector/fields/template-settings.json b/ecs/vulnerability-detector/fields/template-settings.json new file mode 100644 index 0000000000000..b0376b562f775 --- /dev/null +++ b/ecs/vulnerability-detector/fields/template-settings.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["wazuh-states-vulnerabilities"], + "priority": 1, + "template": { + "settings": { + "index": { + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": 2000 + } + } + } + } + } +}