-
Notifications
You must be signed in to change notification settings - Fork 22
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Create vulnerabilities index mapping #6
Create vulnerabilities index mapping #6
Comments
I managed to create the mappings for the fields above using the following command: CommandUsing the ECS generator. WAZUH_INDEXER_ECS=<path-to-wazuh-indexer>/ecs
ECS_VERSION=v8.10.0
python scripts/generator.py --strict --ref $ECS_VERSION \
--subset $WAZUH_INDEXER_ECS/fields/vulnerability-detector/subset.yml \
--template-settings $WAZUH_INDEXER_ECS/fields/vulnerability-detector/template-settings.json \
--template-settings-legacy $WAZUH_INDEXER_ECS/fields/vulnerability-detector/template-settings-legacy.json \
--out $WAZUH_INDEXER_ECS/mappings/$ECS_VERSION Input filessubset.yml
---
name: vulnerability_detector
fields:
base:
fields: "*"
agent:
fields: "*"
ecs:
fields: "*"
event:
fields: "*"
package:
fields: "*"
host:
fields:
os:
fields: "*"
vulnerability:
fields: "*" template-settings-legacy.json
{
"index_patterns": ["wazuh-states-vulnerabilities"],
"order": 1,
"settings": {
"index": {
"codec" : "best_compression",
"mapping": {
"total_fields": {
"limit": 1000
}
},
"refresh_interval": "2s"
}
}
} template-settings.json
{
"index_patterns": ["wazuh-states-vulnerabilities"],
"priority": 1,
"template": {
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": 2000
}
}
}
}
}
} Output files
legacy/template.json
{
"index_patterns": [
"wazuh-states-vulnerabilities"
],
"mappings": {
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"agent": {
"properties": {
"build": {
"properties": {
"original": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ephemeral_id": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"event": {
"properties": {
"action": {
"ignore_above": 1024,
"type": "keyword"
},
"agent_id_status": {
"ignore_above": 1024,
"type": "keyword"
},
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"code": {
"ignore_above": 1024,
"type": "keyword"
},
"created": {
"type": "date"
},
"dataset": {
"ignore_above": 1024,
"type": "keyword"
},
"duration": {
"type": "long"
},
"end": {
"type": "date"
},
"hash": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"ingested": {
"type": "date"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"original": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"outcome": {
"ignore_above": 1024,
"type": "keyword"
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"reason": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"risk_score": {
"type": "float"
},
"risk_score_norm": {
"type": "float"
},
"sequence": {
"type": "long"
},
"severity": {
"type": "long"
},
"start": {
"type": "date"
},
"timezone": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"url": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"os": {
"properties": {
"family": {
"ignore_above": 1024,
"type": "keyword"
},
"full": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"kernel": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"platform": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"labels": {
"type": "object"
},
"message": {
"type": "text"
},
"package": {
"properties": {
"architecture": {
"ignore_above": 1024,
"type": "keyword"
},
"build_version": {
"ignore_above": 1024,
"type": "keyword"
},
"checksum": {
"ignore_above": 1024,
"type": "keyword"
},
"description": {
"ignore_above": 1024,
"type": "keyword"
},
"install_scope": {
"ignore_above": 1024,
"type": "keyword"
},
"installed": {
"type": "date"
},
"license": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"size": {
"type": "long"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"vulnerability": {
"properties": {
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"classification": {
"ignore_above": 1024,
"type": "keyword"
},
"description": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"enumeration": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"report_id": {
"ignore_above": 1024,
"type": "keyword"
},
"scanner": {
"properties": {
"vendor": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"score": {
"properties": {
"base": {
"type": "float"
},
"environmental": {
"type": "float"
},
"temporal": {
"type": "float"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"severity": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"order": 1,
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": 1000
}
},
"refresh_interval": "2s"
}
}
} |
Type Present in |
Loaded the index template using components (composable index template). All the components must be loaded before the template. Loading the legacy template didn't work out-of-the-box, as the schema used by OpenSearch does not match the template from ECS. I tried using an older version of ECS (v1.12.2), but it didn't work either (the same template was generated). {
"error": {
"root_cause": [
{
"type": "x_content_parse_exception",
"reason": "[5:3] [index_template] unknown field [mappings]"
}
],
"type": "x_content_parse_exception",
"reason": "[5:3] [index_template] unknown field [mappings]"
},
"status": 400
} Resources: |
I achieved to load documents to an index that uses the generated index template. Documents attached. event_00_login.json
{
"@timestamp": "2023-09-13T12:34:56.789Z",
"agent": {
"build": {
"original": "build1234"
},
"ephemeral_id": "12345678",
"id": "agent123",
"name": "example-agent",
"type": "agent-type",
"version": "1.2.3"
},
"ecs": {
"version": "1.7.0"
},
"event": {
"action": "login",
"agent_id_status": "status-ok",
"category": "authentication",
"code": "auth-success",
"created": "2023-09-13T12:34:56.789Z",
"dataset": "auth-logs",
"duration": 500,
"end": "2023-09-13T12:35:01.289Z",
"hash": "abc123",
"id": "event123",
"ingested": "2023-09-13T12:36:00.000Z",
"kind": "event-kind",
"module": "security",
"original": "original-event",
"outcome": "success",
"provider": "provider-name",
"reason": "login-reason",
"reference": "event-ref-123",
"risk_score": 7.5,
"risk_score_norm": 0.75,
"sequence": 123456,
"severity": 2,
"start": "2023-09-13T12:34:56.789Z",
"timezone": "UTC",
"type": "event-type",
"url": "https://example.com/event"
},
"labels": {
"label1": "value1",
"label2": "value2"
},
"message": "This is a sample log message.",
"package": {
"architecture": "x86_64",
"build_version": "build123",
"checksum": "checksum123",
"description": "Package description",
"install_scope": "user",
"installed": "2023-09-13T12:34:56.789Z",
"license": "MIT",
"name": "example-package",
"path": "/path/to/package",
"reference": "package-ref-123",
"size": 1024,
"type": "rpm",
"version": "2.0.1"
},
"tags": ["tag1", "tag2"],
"vulnerability": {
"category": "security",
"classification": "high",
"description": "A sample vulnerability",
"enumeration": "CVE-12345",
"id": "vuln123",
"reference": "vulnerability-ref-123",
"report_id": "report-456",
"scanner": {
"vendor": "security-scanner"
},
"score": {
"base": 8.7,
"environmental": 7.5,
"temporal": 9.2,
"version": "1.0"
},
"severity": "critical"
}
} event_01_logout.json
{
"@timestamp": "2023-09-13T13:45:00.000Z",
"agent": {
"name": "another-agent",
"version": "2.1.0"
},
"ecs": {
"version": "1.8.0"
},
"event": {
"action": "logout",
"created": "2023-09-13T13:45:00.000Z",
"category": "system",
"duration": 300,
"outcome": "failure",
"severity": 3
},
"labels": {
"label3": "value3",
"label4": "value4"
},
"message": "User logout failed due to invalid credentials.",
"package": {
"name": "new-package",
"version": "3.0.0"
},
"tags": ["tag3", "tag4"],
"vulnerability": {
"category": "security",
"classification": "medium",
"description": "A different vulnerability",
"enumeration": "CVE-54321",
"id": "vuln456",
"score": {
"base": 6.2,
"environmental": 5.9,
"temporal": 7.1,
"version": "1.5"
},
"severity": "high"
}
} event_01_restart.json
{
"@timestamp": "2023-09-13T14:00:00.000Z",
"agent": {
"name": "linux-agent",
"version": "3.0.0"
},
"ecs": {
"version": "1.9.0"
},
"event": {
"action": "system-restart",
"created": "2023-09-13T14:00:00.000Z",
"category": "system",
"duration": 600,
"outcome": "success",
"severity": 2
},
"os": {
"name": "Linux",
"version": "5.10.0-123.el8.x86_64"
},
"labels": {
"label5": "value5",
"label6": "value6"
},
"message": "System restarted successfully.",
"package": {
"name": "linux-package",
"version": "4.0.0"
},
"tags": ["tag5", "tag6"],
"vulnerability": {
"category": "security",
"classification": "low",
"description": "Linux vulnerability",
"enumeration": "CVE-99999",
"id": "vuln789",
"score": {
"base": 3.5,
"environmental": 4.1,
"temporal": 3.2,
"version": "2.0"
},
"severity": "medium"
}
} |
When you have it available, you can create a PR as part of the SDL in wazuh/wazuh in the branch The folder to upload is |
ECS mappings for Wazuh's vulnerability detectorFolder structure
How to use
Move into the curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_base" -H "Content-Type: application/json" -d @base.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_agent" -H "Content-Type: application/json" -d @agent.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_ecs" -H "Content-Type: application/json" -d @ecs.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_event" -H "Content-Type: application/json" -d @event.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_host" -H "Content-Type: application/json" -d @host.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_package" -H "Content-Type: application/json" -d @package.json
curl -u admin:admin -k -X PUT "https://localhost:9200/_component_template/ecs_8.10.0_vulnerability" -H "Content-Type: application/json" -d @vulnerability.json
Move back to the main folder, where curl -u admin:admin -k -X PUT "https://localhost:9200/_index_template/wazuh-vulnerability-detector" -H "Content-Type: application/json" -d @template.json |
As requested, here's the legacy template: Details
{
"index_patterns": [
"wazuh-states-vulnerabilities"
],
"priority": 1,
"template": {
"mappings": {
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"agent": {
"properties": {
"build": {
"properties": {
"original": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ephemeral_id": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"event": {
"properties": {
"action": {
"ignore_above": 1024,
"type": "keyword"
},
"agent_id_status": {
"ignore_above": 1024,
"type": "keyword"
},
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"code": {
"ignore_above": 1024,
"type": "keyword"
},
"created": {
"type": "date"
},
"dataset": {
"ignore_above": 1024,
"type": "keyword"
},
"duration": {
"type": "long"
},
"end": {
"type": "date"
},
"hash": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"ingested": {
"type": "date"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"original": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"outcome": {
"ignore_above": 1024,
"type": "keyword"
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"reason": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"risk_score": {
"type": "float"
},
"risk_score_norm": {
"type": "float"
},
"sequence": {
"type": "long"
},
"severity": {
"type": "long"
},
"start": {
"type": "date"
},
"timezone": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"url": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"os": {
"properties": {
"family": {
"ignore_above": 1024,
"type": "keyword"
},
"full": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"kernel": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"platform": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"labels": {
"type": "object"
},
"message": {
"type": "text"
},
"package": {
"properties": {
"architecture": {
"ignore_above": 1024,
"type": "keyword"
},
"build_version": {
"ignore_above": 1024,
"type": "keyword"
},
"checksum": {
"ignore_above": 1024,
"type": "keyword"
},
"description": {
"ignore_above": 1024,
"type": "keyword"
},
"install_scope": {
"ignore_above": 1024,
"type": "keyword"
},
"installed": {
"type": "date"
},
"license": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"size": {
"type": "long"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"vulnerability": {
"properties": {
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"classification": {
"ignore_above": 1024,
"type": "keyword"
},
"description": {
"fields": {
"text": {
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"enumeration": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"reference": {
"ignore_above": 1024,
"type": "keyword"
},
"report_id": {
"ignore_above": 1024,
"type": "keyword"
},
"scanner": {
"properties": {
"vendor": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"score": {
"properties": {
"base": {
"type": "float"
},
"environmental": {
"type": "float"
},
"temporal": {
"type": "float"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"severity": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": 1000
}
},
"refresh_interval": "2s"
}
}
}
} Note As stated before, the legacy template generated by the ECS generator can't be used out-of-the-box (see #6 (comment) for more information). Some transformations are required to make this template work, which consists of different names and indentation levels of the JSON keys:
These are requirements from OpenSearch.
|
Pull request created wazuh/wazuh#19015 |
Re-opening due to an issue with the As noted in the Elastic documentation, the operative system fields are not a root level of events, and are meant to be nested under I'm updating the comments in this issue to reflex this change, and also adding a pull request so all the files and tools are hosted and public to everyone. Legacy template updated: #6 (comment) |
Description
We're working on a new vulnerability detector module (see wazuh/wazuh#14153) which will upload all the vulnerabilities it finds into
wazuh-indexer
.The module will create an index and set it up with a mappings' template.
We want to adopt the ECS fields as our schema for this index, so it must contain ECS valid documents only. We will support the following fields in the vulnerability index:
The description for each field is described in https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html and the mappings' definition for each field is in the repository https://github.com/elastic/ecs
The documents must be indexed by an unique key. The initial proposal is to create one based on:
env-name + worker-name + agent-id + cve-id
But we might change this when testing the scalability of the index.
Implementation restrictions
wazuh-indexer
The text was updated successfully, but these errors were encountered: