From 5fc9da46ade9a7d1be54ea9424d920d1ba4d0074 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 15 Jan 2021 13:35:36 -0500 Subject: [PATCH 1/2] Add ssdeep hash (#1169) # Conflicts: # experimental/generated/csv/fields.csv # generated/csv/fields.csv --- CHANGELOG.next.md | 1 + code/go/ecs/hash.go | 9 ++- docs/field-details.asciidoc | 20 +++++- experimental/generated/beats/fields.ecs.yml | 39 +++++++++++- experimental/generated/csv/fields.csv | 4 ++ experimental/generated/ecs/ecs_flat.yml | 44 +++++++++++++ experimental/generated/ecs/ecs_nested.yml | 63 ++++++++++++++++++- .../generated/elasticsearch/7/template.json | 16 +++++ .../elasticsearch/component/dll.json | 4 ++ .../elasticsearch/component/file.json | 4 ++ .../elasticsearch/component/process.json | 8 +++ generated/beats/fields.ecs.yml | 39 +++++++++++- generated/csv/fields.csv | 4 ++ generated/ecs/ecs_flat.yml | 44 +++++++++++++ generated/ecs/ecs_nested.yml | 63 ++++++++++++++++++- generated/elasticsearch/6/template.json | 16 +++++ generated/elasticsearch/7/template.json | 16 +++++ generated/elasticsearch/component/dll.json | 4 ++ generated/elasticsearch/component/file.json | 4 ++ .../elasticsearch/component/process.json | 8 +++ schemas/hash.yml | 11 +++- 21 files changed, 410 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 7abb10dbfb..980907ea27 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -40,6 +40,7 @@ Thanks, you're awesome :-) --> * Added usage documentation for `user` fields. #1066 * Added `user` fields at `user.effective.*`, `user.target.*` and `user.changes.*`. #1066 * Added `os.type`. #1111 +* Added `hash.ssdeep`. #1169 #### Improvements diff --git a/code/go/ecs/hash.go b/code/go/ecs/hash.go index 070b4256cc..aa9354c759 100644 --- a/code/go/ecs/hash.go +++ b/code/go/ecs/hash.go @@ -19,10 +19,14 @@ package ecs -// The hash fields represent different hash algorithms and their values. +// The hash fields represent different bitwise hash algorithms and their +// values. // Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields // for other hashes by lowercasing the hash algorithm name and using underscore // separators as appropriate (snake case, e.g. sha3_512). +// Note that this fieldset is used for common hashes that may be computed over +// a range of generic bytes. Entity-specific hashes such as ja3 or imphash are +// placed in the fieldsets to which they relate (tls and pe, respectively). type Hash struct { // MD5 hash. Md5 string `ecs:"md5"` @@ -35,4 +39,7 @@ type Hash struct { // SHA512 hash. Sha512 string `ecs:"sha512"` + + // SSDEEP hash. + Ssdeep string `ecs:"ssdeep"` } diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index eb72c4aa44..efacab4e1b 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -3023,10 +3023,12 @@ Note also that the `group` fields may be used directly at the root of the events [[ecs-hash]] === Hash Fields -The hash fields represent different hash algorithms and their values. +The hash fields represent different bitwise hash algorithms and their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators as appropriate (snake case, e.g. sha3_512). +Note that this fieldset is used for common hashes that may be computed over a range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed in the fieldsets to which they relate (tls and pe, respectively). + [discrete] ==== Hash Field Details @@ -3096,6 +3098,22 @@ type: keyword +| extended + +// =============================================================== + +| +[[field-hash-ssdeep]] +<> + +| SSDEEP hash. + +type: keyword + + + + + | extended // =============================================================== diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index e687969bf8..23727e859f 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -951,6 +951,12 @@ ignore_above: 1024 description: SHA512 hash. default_field: false + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: name level: core type: keyword @@ -1682,6 +1688,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: inode level: extended type: keyword @@ -2068,11 +2080,16 @@ - name: hash title: Hash group: 2 - description: 'The hash fields represent different hash algorithms and their values. + description: 'The hash fields represent different bitwise hash algorithms and + their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512).' + as appropriate (snake case, e.g. sha3_512). + + Note that this fieldset is used for common hashes that may be computed over + a range of generic bytes. Entity-specific hashes such as ja3 or imphash are + placed in the fieldsets to which they relate (tls and pe, respectively).' type: group fields: - name: md5 @@ -2095,6 +2112,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: host title: Host group: 2 @@ -3500,6 +3523,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: name level: extended type: wildcard @@ -3645,6 +3674,12 @@ ignore_above: 1024 description: SHA512 hash. default_field: false + - name: parent.hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: parent.name level: extended type: wildcard diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index bb4d0bf393..d7cded544d 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -108,6 +108,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev+exp,true,dll,dll.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev+exp,true,dll,dll.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev+exp,true,dll,dll.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev+exp,true,dll,dll.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev+exp,true,dll,dll.name,keyword,core,,kernel32.dll,Name of the library. 1.9.0-dev+exp,true,dll,dll.path,keyword,extended,,C:\Windows\System32\kernel32.dll,Full file path of the library. 1.9.0-dev+exp,true,dll,dll.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. @@ -186,6 +187,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev+exp,true,file,file.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev+exp,true,file,file.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev+exp,true,file,file.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev+exp,true,file,file.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev+exp,true,file,file.inode,keyword,extended,,256383,Inode representing the file in the filesystem. 1.9.0-dev+exp,true,file,file.mime_type,keyword,extended,,,"Media type of file, document, or arrangement of bytes." 1.9.0-dev+exp,true,file,file.mode,keyword,extended,,0640,Mode of the file in octal representation. @@ -395,6 +397,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev+exp,true,process,process.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev+exp,true,process,process.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev+exp,true,process,process.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev+exp,true,process,process.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev+exp,true,process,process.name,wildcard,extended,,ssh,Process name. 1.9.0-dev+exp,true,process,process.name.text,text,extended,,ssh,Process name. 1.9.0-dev+exp,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. @@ -414,6 +417,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev+exp,true,process,process.parent.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev+exp,true,process,process.parent.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev+exp,true,process,process.parent.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev+exp,true,process,process.parent.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev+exp,true,process,process.parent.name,wildcard,extended,,ssh,Process name. 1.9.0-dev+exp,true,process,process.parent.name.text,text,extended,,ssh,Process name. 1.9.0-dev+exp,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 712c63c9a5..ce8874fbe8 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1298,6 +1298,17 @@ dll.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +dll.hash.ssdeep: + dashed_name: dll-hash-ssdeep + description: SSDEEP hash. + flat_name: dll.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword dll.name: dashed_name: dll-name description: 'Name of the library. @@ -2722,6 +2733,17 @@ file.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +file.hash.ssdeep: + dashed_name: file-hash-ssdeep + description: SSDEEP hash. + flat_name: file.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword file.inode: dashed_name: file-inode description: Inode representing the file in the filesystem. @@ -5283,6 +5305,17 @@ process.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +process.hash.ssdeep: + dashed_name: process-hash-ssdeep + description: SSDEEP hash. + flat_name: process.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. @@ -5518,6 +5551,17 @@ process.parent.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +process.parent.hash.ssdeep: + dashed_name: process-parent-hash-ssdeep + description: SSDEEP hash. + flat_name: process.parent.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.parent.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 9016fe8641..fcd725bb6b 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1644,6 +1644,17 @@ dll: original_fieldset: hash short: SHA512 hash. type: keyword + dll.hash.ssdeep: + dashed_name: dll-hash-ssdeep + description: SSDEEP hash. + flat_name: dll.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword dll.name: dashed_name: dll-name description: 'Name of the library. @@ -3170,6 +3181,17 @@ file: original_fieldset: hash short: SHA512 hash. type: keyword + file.hash.ssdeep: + dashed_name: file-hash-ssdeep + description: SSDEEP hash. + flat_name: file.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword file.inode: dashed_name: file-inode description: Inode representing the file in the filesystem. @@ -3902,11 +3924,16 @@ group: title: Group type: group hash: - description: 'The hash fields represent different hash algorithms and their values. + description: 'The hash fields represent different bitwise hash algorithms and their + values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512).' + as appropriate (snake case, e.g. sha3_512). + + Note that this fieldset is used for common hashes that may be computed over a + range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed + in the fieldsets to which they relate (tls and pe, respectively).' fields: hash.md5: dashed_name: hash-md5 @@ -3948,6 +3975,16 @@ hash: normalize: [] short: SHA512 hash. type: keyword + hash.ssdeep: + dashed_name: hash-ssdeep + description: SSDEEP hash. + flat_name: hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + short: SSDEEP hash. + type: keyword group: 2 name: hash prefix: hash. @@ -6379,6 +6416,17 @@ process: original_fieldset: hash short: SHA512 hash. type: keyword + process.hash.ssdeep: + dashed_name: process-hash-ssdeep + description: SSDEEP hash. + flat_name: process.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. @@ -6614,6 +6662,17 @@ process: original_fieldset: hash short: SHA512 hash. type: keyword + process.parent.hash.ssdeep: + dashed_name: process-parent-hash-ssdeep + description: SSDEEP hash. + flat_name: process.parent.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.parent.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. diff --git a/experimental/generated/elasticsearch/7/template.json b/experimental/generated/elasticsearch/7/template.json index 1582968a6f..aebee4c182 100644 --- a/experimental/generated/elasticsearch/7/template.json +++ b/experimental/generated/elasticsearch/7/template.json @@ -523,6 +523,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -853,6 +857,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1843,6 +1851,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1927,6 +1939,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/experimental/generated/elasticsearch/component/dll.json b/experimental/generated/elasticsearch/component/dll.json index 7491296fa2..f791052452 100644 --- a/experimental/generated/elasticsearch/component/dll.json +++ b/experimental/generated/elasticsearch/component/dll.json @@ -46,6 +46,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/experimental/generated/elasticsearch/component/file.json b/experimental/generated/elasticsearch/component/file.json index 58379893c1..0ae17a7b92 100644 --- a/experimental/generated/elasticsearch/component/file.json +++ b/experimental/generated/elasticsearch/component/file.json @@ -82,6 +82,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/experimental/generated/elasticsearch/component/process.json b/experimental/generated/elasticsearch/component/process.json index 9fad9bcc0c..ed0330dafa 100644 --- a/experimental/generated/elasticsearch/component/process.json +++ b/experimental/generated/elasticsearch/component/process.json @@ -78,6 +78,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -162,6 +166,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index d31d592579..75b3f4a862 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -899,6 +899,12 @@ ignore_above: 1024 description: SHA512 hash. default_field: false + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: name level: core type: keyword @@ -1630,6 +1636,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: inode level: extended type: keyword @@ -2016,11 +2028,16 @@ - name: hash title: Hash group: 2 - description: 'The hash fields represent different hash algorithms and their values. + description: 'The hash fields represent different bitwise hash algorithms and + their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512).' + as appropriate (snake case, e.g. sha3_512). + + Note that this fieldset is used for common hashes that may be computed over + a range of generic bytes. Entity-specific hashes such as ja3 or imphash are + placed in the fieldsets to which they relate (tls and pe, respectively).' type: group fields: - name: md5 @@ -2043,6 +2060,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: host title: Host group: 2 @@ -3402,6 +3425,12 @@ type: keyword ignore_above: 1024 description: SHA512 hash. + - name: hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: name level: extended type: wildcard @@ -3547,6 +3576,12 @@ ignore_above: 1024 description: SHA512 hash. default_field: false + - name: parent.hash.ssdeep + level: extended + type: keyword + ignore_above: 1024 + description: SSDEEP hash. + default_field: false - name: parent.name level: extended type: wildcard diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 0d13e795b4..68c90ecb74 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -105,6 +105,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev,true,dll,dll.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev,true,dll,dll.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev,true,dll,dll.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev,true,dll,dll.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev,true,dll,dll.name,keyword,core,,kernel32.dll,Name of the library. 1.9.0-dev,true,dll,dll.path,keyword,extended,,C:\Windows\System32\kernel32.dll,Full file path of the library. 1.9.0-dev,true,dll,dll.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. @@ -183,6 +184,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev,true,file,file.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev,true,file,file.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev,true,file,file.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev,true,file,file.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev,true,file,file.inode,keyword,extended,,256383,Inode representing the file in the filesystem. 1.9.0-dev,true,file,file.mime_type,keyword,extended,,,"Media type of file, document, or arrangement of bytes." 1.9.0-dev,true,file,file.mode,keyword,extended,,0640,Mode of the file in octal representation. @@ -385,6 +387,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev,true,process,process.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev,true,process,process.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev,true,process,process.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev,true,process,process.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev,true,process,process.name,wildcard,extended,,ssh,Process name. 1.9.0-dev,true,process,process.name.text,text,extended,,ssh,Process name. 1.9.0-dev,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. @@ -404,6 +407,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.9.0-dev,true,process,process.parent.hash.sha1,keyword,extended,,,SHA1 hash. 1.9.0-dev,true,process,process.parent.hash.sha256,keyword,extended,,,SHA256 hash. 1.9.0-dev,true,process,process.parent.hash.sha512,keyword,extended,,,SHA512 hash. +1.9.0-dev,true,process,process.parent.hash.ssdeep,keyword,extended,,,SSDEEP hash. 1.9.0-dev,true,process,process.parent.name,wildcard,extended,,ssh,Process name. 1.9.0-dev,true,process,process.parent.name.text,text,extended,,ssh,Process name. 1.9.0-dev,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 73efa64c18..f30e8aced4 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1252,6 +1252,17 @@ dll.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +dll.hash.ssdeep: + dashed_name: dll-hash-ssdeep + description: SSDEEP hash. + flat_name: dll.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword dll.name: dashed_name: dll-name description: 'Name of the library. @@ -2676,6 +2687,17 @@ file.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +file.hash.ssdeep: + dashed_name: file-hash-ssdeep + description: SSDEEP hash. + flat_name: file.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword file.inode: dashed_name: file-inode description: Inode representing the file in the filesystem. @@ -5163,6 +5185,17 @@ process.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +process.hash.ssdeep: + dashed_name: process-hash-ssdeep + description: SSDEEP hash. + flat_name: process.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. @@ -5398,6 +5431,17 @@ process.parent.hash.sha512: original_fieldset: hash short: SHA512 hash. type: keyword +process.parent.hash.ssdeep: + dashed_name: process-parent-hash-ssdeep + description: SSDEEP hash. + flat_name: process.parent.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.parent.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 1461638964..8c15d879d4 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1575,6 +1575,17 @@ dll: original_fieldset: hash short: SHA512 hash. type: keyword + dll.hash.ssdeep: + dashed_name: dll-hash-ssdeep + description: SSDEEP hash. + flat_name: dll.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword dll.name: dashed_name: dll-name description: 'Name of the library. @@ -3101,6 +3112,17 @@ file: original_fieldset: hash short: SHA512 hash. type: keyword + file.hash.ssdeep: + dashed_name: file-hash-ssdeep + description: SSDEEP hash. + flat_name: file.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword file.inode: dashed_name: file-inode description: Inode representing the file in the filesystem. @@ -3833,11 +3855,16 @@ group: title: Group type: group hash: - description: 'The hash fields represent different hash algorithms and their values. + description: 'The hash fields represent different bitwise hash algorithms and their + values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators - as appropriate (snake case, e.g. sha3_512).' + as appropriate (snake case, e.g. sha3_512). + + Note that this fieldset is used for common hashes that may be computed over a + range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed + in the fieldsets to which they relate (tls and pe, respectively).' fields: hash.md5: dashed_name: hash-md5 @@ -3879,6 +3906,16 @@ hash: normalize: [] short: SHA512 hash. type: keyword + hash.ssdeep: + dashed_name: hash-ssdeep + description: SSDEEP hash. + flat_name: hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + short: SSDEEP hash. + type: keyword group: 2 name: hash prefix: hash. @@ -6236,6 +6273,17 @@ process: original_fieldset: hash short: SHA512 hash. type: keyword + process.hash.ssdeep: + dashed_name: process-hash-ssdeep + description: SSDEEP hash. + flat_name: process.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. @@ -6471,6 +6519,17 @@ process: original_fieldset: hash short: SHA512 hash. type: keyword + process.parent.hash.ssdeep: + dashed_name: process-parent-hash-ssdeep + description: SSDEEP hash. + flat_name: process.parent.hash.ssdeep + ignore_above: 1024 + level: extended + name: ssdeep + normalize: [] + original_fieldset: hash + short: SSDEEP hash. + type: keyword process.parent.name: beta: Note the usage of `wildcard` type is considered beta. This field used to be type `keyword`. diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index 6b84ad5897..5d91ba5198 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -526,6 +526,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -864,6 +868,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1827,6 +1835,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1914,6 +1926,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 2f02eb6e41..85c3f90970 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -510,6 +510,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -840,6 +844,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1780,6 +1788,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1864,6 +1876,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/generated/elasticsearch/component/dll.json b/generated/elasticsearch/component/dll.json index 8e878c310e..5c4ff06d3f 100644 --- a/generated/elasticsearch/component/dll.json +++ b/generated/elasticsearch/component/dll.json @@ -46,6 +46,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/generated/elasticsearch/component/file.json b/generated/elasticsearch/component/file.json index a2f17562f9..10b9ed8f62 100644 --- a/generated/elasticsearch/component/file.json +++ b/generated/elasticsearch/component/file.json @@ -82,6 +82,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/generated/elasticsearch/component/process.json b/generated/elasticsearch/component/process.json index 60ad49260b..f214a3c6bd 100644 --- a/generated/elasticsearch/component/process.json +++ b/generated/elasticsearch/component/process.json @@ -78,6 +78,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -162,6 +166,10 @@ "sha512": { "ignore_above": 1024, "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/schemas/hash.yml b/schemas/hash.yml index cc44dfcc8b..77aeb29a5d 100644 --- a/schemas/hash.yml +++ b/schemas/hash.yml @@ -5,12 +5,16 @@ type: group short: Hashes, usually file hashes. description: > - The hash fields represent different hash algorithms and their values. + The hash fields represent different bitwise hash algorithms and their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators as appropriate (snake case, e.g. sha3_512). + Note that this fieldset is used for common hashes that may be computed + over a range of generic bytes. Entity-specific hashes such as ja3 or imphash are + placed in the fieldsets to which they relate (tls and pe, respectively). + reusable: top_level: false expected: @@ -39,3 +43,8 @@ level: extended type: keyword description: SHA512 hash. + + - name: ssdeep + level: extended + type: keyword + description: SSDEEP hash. From a9a8470eaa53aadaa04ee032fcd78ac2e92faec9 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Fri, 15 Jan 2021 12:44:21 -0600 Subject: [PATCH 2/2] move changelog entry to correct section --- CHANGELOG.next.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 980907ea27..9baec5b634 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -19,6 +19,7 @@ Thanks, you're awesome :-) --> * Added `http.request.id`. #1208 * Added `cloud.service.name`. #1204 +* Added `hash.ssdeep`. #1169 #### Improvements @@ -40,7 +41,6 @@ Thanks, you're awesome :-) --> * Added usage documentation for `user` fields. #1066 * Added `user` fields at `user.effective.*`, `user.target.*` and `user.changes.*`. #1066 * Added `os.type`. #1111 -* Added `hash.ssdeep`. #1169 #### Improvements