diff --git a/Makefile b/Makefile index 67ee219d8a..327f64b49f 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ check-license-headers: # Clean deletes all temporary and generated content. .PHONY: clean clean: - rm -rf build + rm -rf build generated/elasticsearch/component experimental/generated/elasticsearch/component # Clean all markdown files for use-cases find ./use-cases -type f -name '*.md' -not -name 'README.md' -print0 | xargs -0 rm -- diff --git a/experimental/generated/elasticsearch/component/as.json b/experimental/generated/elasticsearch/component/as.json deleted file mode 100644 index 85b465e03e..0000000000 --- a/experimental/generated/elasticsearch/component/as.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "as": { - "properties": { - "number": { - "type": "long" - }, - "organization": { - "properties": { - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "type": "wildcard" - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/code_signature.json b/experimental/generated/elasticsearch/component/code_signature.json deleted file mode 100644 index 66c183f3f7..0000000000 --- a/experimental/generated/elasticsearch/component/code_signature.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "code_signature": { - "properties": { - "exists": { - "type": "boolean" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - }, - "subject_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/geo.json b/experimental/generated/elasticsearch/component/geo.json deleted file mode 100644 index 81dc5defcb..0000000000 --- a/experimental/generated/elasticsearch/component/geo.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "type": "wildcard" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/hash.json b/experimental/generated/elasticsearch/component/hash.json deleted file mode 100644 index d7776dcf12..0000000000 --- a/experimental/generated/elasticsearch/component/hash.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "hash": { - "properties": { - "md5": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/interface.json b/experimental/generated/elasticsearch/component/interface.json deleted file mode 100644 index 67b95e8dc9..0000000000 --- a/experimental/generated/elasticsearch/component/interface.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "interface": { - "properties": { - "alias": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/os.json b/experimental/generated/elasticsearch/component/os.json deleted file mode 100644 index db3eca753d..0000000000 --- a/experimental/generated/elasticsearch/component/os.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "type": "wildcard" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "type": "wildcard" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/pe.json b/experimental/generated/elasticsearch/component/pe.json deleted file mode 100644 index b5ca655c97..0000000000 --- a/experimental/generated/elasticsearch/component/pe.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "pe": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "type": "wildcard" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/vlan.json b/experimental/generated/elasticsearch/component/vlan.json deleted file mode 100644 index 3cddf5221a..0000000000 --- a/experimental/generated/elasticsearch/component/vlan.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "vlan": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/component/x509.json b/experimental/generated/elasticsearch/component/x509.json deleted file mode 100644 index d6ea1c8df7..0000000000 --- a/experimental/generated/elasticsearch/component/x509.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "x509": { - "properties": { - "alternative_names": { - "ignore_above": 1024, - "type": "keyword" - }, - "issuer": { - "properties": { - "common_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country": { - "ignore_above": 1024, - "type": "keyword" - }, - "distinguished_name": { - "type": "wildcard" - }, - "locality": { - "ignore_above": 1024, - "type": "keyword" - }, - "organization": { - "ignore_above": 1024, - "type": "keyword" - }, - "organizational_unit": { - "ignore_above": 1024, - "type": "keyword" - }, - "state_or_province": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "not_after": { - "type": "date" - }, - "not_before": { - "type": "date" - }, - "public_key_algorithm": { - "ignore_above": 1024, - "type": "keyword" - }, - "public_key_curve": { - "ignore_above": 1024, - "type": "keyword" - }, - "public_key_exponent": { - "doc_values": false, - "index": false, - "type": "long" - }, - "public_key_size": { - "type": "long" - }, - "serial_number": { - "ignore_above": 1024, - "type": "keyword" - }, - "signature_algorithm": { - "ignore_above": 1024, - "type": "keyword" - }, - "subject": { - "properties": { - "common_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country": { - "ignore_above": 1024, - "type": "keyword" - }, - "distinguished_name": { - "type": "wildcard" - }, - "locality": { - "ignore_above": 1024, - "type": "keyword" - }, - "organization": { - "ignore_above": 1024, - "type": "keyword" - }, - "organizational_unit": { - "ignore_above": 1024, - "type": "keyword" - }, - "state_or_province": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "version_number": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/experimental/generated/elasticsearch/template.json b/experimental/generated/elasticsearch/template.json index ba5614931b..41ebc2ac37 100644 --- a/experimental/generated/elasticsearch/template.json +++ b/experimental/generated/elasticsearch/template.json @@ -5,11 +5,9 @@ }, "composed_of": [ "ecs_2.0.0-dev-exp_agent", - "ecs_2.0.0-dev-exp_as", "ecs_2.0.0-dev-exp_base", "ecs_2.0.0-dev-exp_client", "ecs_2.0.0-dev-exp_cloud", - "ecs_2.0.0-dev-exp_code_signature", "ecs_2.0.0-dev-exp_container", "ecs_2.0.0-dev-exp_destination", "ecs_2.0.0-dev-exp_dll", @@ -18,19 +16,14 @@ "ecs_2.0.0-dev-exp_error", "ecs_2.0.0-dev-exp_event", "ecs_2.0.0-dev-exp_file", - "ecs_2.0.0-dev-exp_geo", "ecs_2.0.0-dev-exp_group", - "ecs_2.0.0-dev-exp_hash", "ecs_2.0.0-dev-exp_host", "ecs_2.0.0-dev-exp_http", - "ecs_2.0.0-dev-exp_interface", "ecs_2.0.0-dev-exp_log", "ecs_2.0.0-dev-exp_network", "ecs_2.0.0-dev-exp_observer", "ecs_2.0.0-dev-exp_organization", - "ecs_2.0.0-dev-exp_os", "ecs_2.0.0-dev-exp_package", - "ecs_2.0.0-dev-exp_pe", "ecs_2.0.0-dev-exp_process", "ecs_2.0.0-dev-exp_registry", "ecs_2.0.0-dev-exp_related", @@ -44,9 +37,7 @@ "ecs_2.0.0-dev-exp_url", "ecs_2.0.0-dev-exp_user", "ecs_2.0.0-dev-exp_user_agent", - "ecs_2.0.0-dev-exp_vlan", - "ecs_2.0.0-dev-exp_vulnerability", - "ecs_2.0.0-dev-exp_x509" + "ecs_2.0.0-dev-exp_vulnerability" ], "index_patterns": [ "try-ecs-*" diff --git a/generated/elasticsearch/component/as.json b/generated/elasticsearch/component/as.json deleted file mode 100644 index 1f8f48b729..0000000000 --- a/generated/elasticsearch/component/as.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "as": { - "properties": { - "number": { - "type": "long" - }, - "organization": { - "properties": { - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/code_signature.json b/generated/elasticsearch/component/code_signature.json deleted file mode 100644 index 66c183f3f7..0000000000 --- a/generated/elasticsearch/component/code_signature.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "code_signature": { - "properties": { - "exists": { - "type": "boolean" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - }, - "subject_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/geo.json b/generated/elasticsearch/component/geo.json deleted file mode 100644 index 55cdb3be35..0000000000 --- a/generated/elasticsearch/component/geo.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/hash.json b/generated/elasticsearch/component/hash.json deleted file mode 100644 index d7776dcf12..0000000000 --- a/generated/elasticsearch/component/hash.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "hash": { - "properties": { - "md5": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/interface.json b/generated/elasticsearch/component/interface.json deleted file mode 100644 index 67b95e8dc9..0000000000 --- a/generated/elasticsearch/component/interface.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "interface": { - "properties": { - "alias": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/os.json b/generated/elasticsearch/component/os.json deleted file mode 100644 index 4d23c26134..0000000000 --- a/generated/elasticsearch/component/os.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/pe.json b/generated/elasticsearch/component/pe.json deleted file mode 100644 index bbdae948d1..0000000000 --- a/generated/elasticsearch/component/pe.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "pe": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/vlan.json b/generated/elasticsearch/component/vlan.json deleted file mode 100644 index 3cddf5221a..0000000000 --- a/generated/elasticsearch/component/vlan.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "vlan": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/component/x509.json b/generated/elasticsearch/component/x509.json deleted file mode 100644 index 874b47ffea..0000000000 --- a/generated/elasticsearch/component/x509.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "template": { - "mappings": { - "properties": { - "x509": { - "properties": { - "alternative_names": { - "ignore_above": 1024, - "type": "keyword" - }, - "issuer": { - "properties": { - "common_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country": { - "ignore_above": 1024, - "type": "keyword" - }, - "distinguished_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "locality": { - "ignore_above": 1024, - "type": "keyword" - }, - "organization": { - "ignore_above": 1024, - "type": "keyword" - }, - "organizational_unit": { - "ignore_above": 1024, - "type": "keyword" - }, - "state_or_province": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "not_after": { - "type": "date" - }, - "not_before": { - "type": "date" - }, - "public_key_algorithm": { - "ignore_above": 1024, - "type": "keyword" - }, - "public_key_curve": { - "ignore_above": 1024, - "type": "keyword" - }, - "public_key_exponent": { - "doc_values": false, - "index": false, - "type": "long" - }, - "public_key_size": { - "type": "long" - }, - "serial_number": { - "ignore_above": 1024, - "type": "keyword" - }, - "signature_algorithm": { - "ignore_above": 1024, - "type": "keyword" - }, - "subject": { - "properties": { - "common_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country": { - "ignore_above": 1024, - "type": "keyword" - }, - "distinguished_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "locality": { - "ignore_above": 1024, - "type": "keyword" - }, - "organization": { - "ignore_above": 1024, - "type": "keyword" - }, - "organizational_unit": { - "ignore_above": 1024, - "type": "keyword" - }, - "state_or_province": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "version_number": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/generated/elasticsearch/template.json b/generated/elasticsearch/template.json index 4c51d85b5e..f8e8a919f2 100644 --- a/generated/elasticsearch/template.json +++ b/generated/elasticsearch/template.json @@ -5,11 +5,9 @@ }, "composed_of": [ "ecs_2.0.0-dev_agent", - "ecs_2.0.0-dev_as", "ecs_2.0.0-dev_base", "ecs_2.0.0-dev_client", "ecs_2.0.0-dev_cloud", - "ecs_2.0.0-dev_code_signature", "ecs_2.0.0-dev_container", "ecs_2.0.0-dev_destination", "ecs_2.0.0-dev_dll", @@ -18,19 +16,14 @@ "ecs_2.0.0-dev_error", "ecs_2.0.0-dev_event", "ecs_2.0.0-dev_file", - "ecs_2.0.0-dev_geo", "ecs_2.0.0-dev_group", - "ecs_2.0.0-dev_hash", "ecs_2.0.0-dev_host", "ecs_2.0.0-dev_http", - "ecs_2.0.0-dev_interface", "ecs_2.0.0-dev_log", "ecs_2.0.0-dev_network", "ecs_2.0.0-dev_observer", "ecs_2.0.0-dev_organization", - "ecs_2.0.0-dev_os", "ecs_2.0.0-dev_package", - "ecs_2.0.0-dev_pe", "ecs_2.0.0-dev_process", "ecs_2.0.0-dev_registry", "ecs_2.0.0-dev_related", @@ -44,9 +37,7 @@ "ecs_2.0.0-dev_url", "ecs_2.0.0-dev_user", "ecs_2.0.0-dev_user_agent", - "ecs_2.0.0-dev_vlan", - "ecs_2.0.0-dev_vulnerability", - "ecs_2.0.0-dev_x509" + "ecs_2.0.0-dev_vulnerability" ], "index_patterns": [ "try-ecs-*" diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 4022e666e5..e3e218acc9 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -13,10 +13,10 @@ def generate(ecs_nested, ecs_version, out_dir, mapping_settings_file): """This generates all artifacts for the composable template approach""" all_component_templates(ecs_nested, ecs_version, out_dir) component_names = component_name_convention(ecs_version, ecs_nested) - composable_template(ecs_version, component_names, out_dir, mapping_settings_file) + save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file) -def composable_template(ecs_version, component_names, out_dir, mapping_settings_file): +def save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file): """Generate the master sample composable template""" template = { "index_patterns": ["try-ecs-*"], @@ -48,16 +48,16 @@ def all_component_templates(ecs_nested, ecs_version, out_dir): component_dir = join(out_dir, 'elasticsearch/component') ecs_helpers.make_dirs(component_dir) - for (fieldset_name, fieldset) in ecs_nested.items(): + for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): field_mappings = {} for (flat_name, field) in fieldset['fields'].items(): name_parts = flat_name.split('.') dict_add_nested(field_mappings, name_parts, entry_for(field)) - component_template(fieldset_name, ecs_version, component_dir, field_mappings) + save_component_template(fieldset_name, ecs_version, component_dir, field_mappings) -def component_template(template_name, ecs_version, out_dir, field_mappings): +def save_component_template(template_name, ecs_version, out_dir, field_mappings): filename = join(out_dir, template_name) + ".json" template = {'template': {'mappings': {'properties': field_mappings}}} @@ -67,11 +67,22 @@ def component_template(template_name, ecs_version, out_dir, field_mappings): def component_name_convention(ecs_version, ecs_nested): version = ecs_version.replace('+', '-') names = [] - for (fieldset_name, fieldset) in ecs_nested.items(): + for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): names.append("ecs_{}_{}".format(version, fieldset_name)) return names +def candidate_components(ecs_nested): + """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False""" + components = {} + for (fieldset_name, fieldset) in ecs_nested.items(): + if fieldset.get('reusable', None): + if not fieldset['reusable']['top_level']: + continue + components[fieldset_name] = fieldset + return components + + # Legacy template