diff --git a/big5/README.md b/big5/README.md index 2286a2fa..5d5882d3 100755 --- a/big5/README.md +++ b/big5/README.md @@ -51,6 +51,7 @@ This workload allows the following parameters to be specified using `--workload- * `document_uncompressed_size_in_bytes`: If specifying an alternate data corpus, the uncompressed size of the corpus. * `document_url`: If specifying an alternate data corpus, the full path to the corpus file (optional). * `error_level` (default: "non-fatal"): Available for bulk operations only to specify ignore-response-error-level. +* `index_body` (default: "index.json"): The name of the file containing the index settings and mappings. * `index_name` (default: "big5"): The name of the index the workload should create and use for its operations. * `index_settings`: A list of index settings. Index settings defined elsewhere (e.g. `number_of_replicas`) need to be overridden explicitly. * `ingest_percentage` (default: 100): A number between 0 and 100 that defines how much of the document corpus should be ingested. diff --git a/big5/index.json.ORIG b/big5/index-alt-1.json similarity index 90% rename from big5/index.json.ORIG rename to big5/index-alt-1.json index 6bfb3d2a..a9ebecfa 100755 --- a/big5/index.json.ORIG +++ b/big5/index-alt-1.json @@ -1,8 +1,12 @@ { "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 1, - "index.requests.cache.enable": false + "index.number_of_shards": {{number_of_shards | default(1)}}, + "index.number_of_replicas": {{number_of_replicas | default(1)}}, + "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}}, + "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}}, + "index.codec": "best_compression", + "index.translog.sync_interval": "30s", + "index.translog.durability": "async" }, "mappings": { "_data_stream_timestamp": { diff --git a/big5/index-alt-2.json b/big5/index-alt-2.json new file mode 100644 index 00000000..7c37cc0d --- /dev/null +++ b/big5/index-alt-2.json @@ -0,0 +1,182 @@ +{ + "settings": { + "index.number_of_shards": {{number_of_shards | default(1)}}, + "index.number_of_replicas": {{number_of_replicas | default(1)}}, + "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}}, + "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}}, + "index.codec": "best_compression", + "index.translog.sync_interval": "30s", + "index.translog.durability": "async", + "index.query.default_field": [ "message" ] + }, + "mappings": { + "dynamic_templates": [ + { + "match_ip": { + "match": "ip", + "match_mapping_type": "string", + "mapping": { + "type": "ip" + } + } + }, + { + "match_message": { + "match": "message", + "match_mapping_type": "string", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "time_series_dimension": true + }, + "type": { + "type": "keyword", + "time_series_dimension": true + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "aws": { + "properties": { + "cloudwatch": { + "properties": { + "ingestion_time": { + "type": "keyword", + "ignore_above": 1024 + }, + "log_group": { + "type": "keyword", + "time_series_dimension": true + }, + "log_stream": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + }, + "cloud": { + "properties": { + "region": { + "type": "keyword", + "time_series_dimension": true + } + } + }, + "ecs": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "event": { + "properties": { + "dataset": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "time_series_dimension": true + }, + "ingested": { + "type": "date" + } + } + }, + "host": { + "type": "object" + }, + "input": { + "properties": { + "type": { + "type": "keyword", + "time_series_dimension": true + } + } + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "message": { + "type": "match_only_text" + }, + "meta": { + "properties": { + "file": { + "type": "keyword", + "time_series_dimension": true + } + } + }, + "metrics": { + "properties": { + "size": { + "type": "long", + "time_series_metric": "gauge" + }, + "tmin": { + "type": "long", + "time_series_metric": "gauge" + } + } + }, + "process": { + "properties": { + "name": { + "type": "keyword", + "time_series_dimension": true + } + } + }, + "tags": { + "type": "keyword", + "ignore_above": 1024 + } + } + } +} diff --git a/big5/index.json b/big5/index.json index 115c0a90..15cdd40b 100644 --- a/big5/index.json +++ b/big5/index.json @@ -1,164 +1,165 @@ { - "settings": { - "index.number_of_shards": {{number_of_shards | default(1)}}, - "index.number_of_replicas": {{number_of_replicas | default(1)}}, - "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}}, - "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}}, - "index.codec": "best_compression", - "index.translog.sync_interval": "30s", - "index.translog.durability": "async" + "settings": { + "index.number_of_shards": {{number_of_shards | default(1)}}, + "index.number_of_replicas": {{number_of_replicas | default(1)}}, + "index.queries.cache.enabled": {{query_cache_enabled | default(false) | tojson}}, + "index.requests.cache.enable": {{requests_cache_enabled | default(false) | tojson}}, + "index.codec": "best_compression", + "index.translog.sync_interval": "30s", + "index.translog.durability": "async", + "index.query.default_field": [ "message" ] + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" }, - "mappings": { - "properties": { - "agent": { - "type": "object", - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "type": "object", - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "type": "object", - "properties": { - "file": { - "type": "object", - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "message": { - "type": "text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "cloud": { - "type": "object", - "properties": { - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "input": { - "type": "object", - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "type": "object", - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "meta": { - "type": "object", - "properties": { - "file": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "type": "object" - }, - "metrics": { - "type": "object", - "properties": { - "size": { - "type": "long" - }, - "tmax": { - "type": "long" - }, - "tmin": { - "type": "long" - } - } - }, - "aws": { - "type": "object", - "properties": { - "cloudwatch": { - "type": "object", - "properties": { - "log_group": { - "ignore_above": 1024, - "type": "keyword" - }, - "ingestion_time": { - "ignore_above": 1024, - "type": "keyword" - }, - "log_stream": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "event": { - "type": "object", - "properties": { - "ingested": { - "type": "date" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } + "agent": { + "type": "object", + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "aws": { + "type": "object", + "properties": { + "cloudwatch": { + "type": "object", + "properties": { + "log_group": { + "type": "keyword", + "ignore_above": 1024 + }, + "ingestion_time": { + "type": "keyword", + "ignore_above": 1024 + }, + "log_stream": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "cloud": { + "type": "object", + "properties": { + "region": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "ecs": { + "type": "object", + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "event": { + "type": "object", + "properties": { + "ingested": { + "type": "date" + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "dataset": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "host": { + "type": "object" + }, + "input": { + "type": "object", + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "log": { + "type": "object", + "properties": { + "file": { + "type": "object", + "properties": { + "path": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "message": { + "type": "text" + }, + "meta": { + "type": "object", + "properties": { + "file": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "metrics": { + "type": "object", + "properties": { + "size": { + "type": "long" + }, + "tmax": { + "type": "long" + }, + "tmin": { + "type": "long" + } + } + }, + "process": { + "type": "object", + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "tags": { + "type": "keyword", + "ignore_above": 1024 } + } + } } diff --git a/big5/workload.json b/big5/workload.json index 65f384db..ebc79018 100644 --- a/big5/workload.json +++ b/big5/workload.json @@ -1,12 +1,10 @@ -{% set index_body = 'index.json' %} - { "version": 2, - "description": "Random HTTP server log data", + "description": "Big5 workload based on synthetically generated data corpus", "indices": [ { - "name": "{{index_name | default('big5')}}", - "body": "{{ index_body }}" + "name": "{{ index_name | default('big5') }}", + "body": "{{ index_body | default('index.json') }}" } ], "corpora": [