Make file integrity path field analyzed

CHANGELOG.asciidoc

@@ -18,6 +18,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Auditbeat*
 
+- Changed `audit.file.path` to be a multi-field so that path is searchable. {pull}5625[5625]
+
 *Filebeat*
 
 *Heartbeat*

auditbeat/docs/fields.asciidoc

@@ -43,10 +43,18 @@ The file metricset generates events when a file changes on disk.
 [float]
 === `audit.file.path`
 
-type: keyword
+type: text
 
 The path to the file.
 
+[float]
+=== `audit.file.path.raw`
+
+type: keyword
+
+The path to the file. This is an non-analyzed field that is useful for aggregations.
+
+
 [float]
 === `audit.file.target_path`
 

auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-kernel-executions.json

@@ -29,7 +29,7 @@
       },
       "id": "f81a6de0-c1c1-11e7-8995-936807a28b16",
       "type": "visualization",
-      "version": 2
+      "version": 1
     },
     {
       "attributes": {
@@ -70,7 +70,7 @@
       },
       "id": "d382f5b0-c1c6-11e7-8995-936807a28b16",
       "type": "search",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -88,8 +88,8 @@
       },
       "id": "7de391b0-c1ca-11e7-8995-936807a28b16",
       "type": "dashboard",
-      "version": 3
+      "version": 1
     }
   ],
-  "version": "6.0.0-rc2"
+  "version": "6.0.0"
 }

auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-kernel-overview.json

@@ -13,7 +13,7 @@
       },
       "id": "97680df0-c1c0-11e7-8995-936807a28b16",
       "type": "visualization",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -42,7 +42,7 @@
       },
       "id": "0f10c430-c1c3-11e7-8995-936807a28b16",
       "type": "search",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -75,8 +75,8 @@
       },
       "id": "c0ac2c00-c1c0-11e7-8995-936807a28b16",
       "type": "dashboard",
-      "version": 7
+      "version": 1
     }
   ],
-  "version": "6.0.0-rc2"
+  "version": "6.0.0"
 }

auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-kernel-sockets.json

@@ -14,7 +14,7 @@
       },
       "id": "faf882f0-c242-11e7-8692-232bd1143e8a",
       "type": "visualization",
-      "version": 4
+      "version": 1
     },
     {
       "attributes": {
@@ -30,7 +30,7 @@
       },
       "id": "ea483730-c246-11e7-8692-232bd1143e8a",
       "type": "visualization",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -76,7 +76,7 @@
       },
       "id": "a8e20450-c256-11e7-8692-232bd1143e8a",
       "type": "visualization",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -129,7 +129,7 @@
       },
       "id": "5438b030-c246-11e7-8692-232bd1143e8a",
       "type": "search",
-      "version": 2
+      "version": 1
     },
     {
       "attributes": {
@@ -173,8 +173,8 @@
       },
       "id": "693a5f40-c243-11e7-8692-232bd1143e8a",
       "type": "dashboard",
-      "version": 6
+      "version": 1
     }
   ],
-  "version": "6.0.0-rc2"
+  "version": "6.0.0"
 }

auditbeat/module/audit/file/_meta/fields.yml

@@ -4,89 +4,121 @@
       The file metricset generates events when a file changes on disk.
     fields:
     - name: path
-      type: keyword
+      type: text
       description: The path to the file.
+      multi_fields:
+      - name: raw
+        type: keyword
+        description: >
+          The path to the file. This is an non-analyzed field that is useful
+          for aggregations.
+
     - name: target_path
       type: keyword
       description: The target path for symlinks.
+
     - name: action
       type: keyword
       example: attributes_modified
       description: >
         Action describes the change that triggered the event. The possible
         values are: attributes_modified, created, deleted, updated, moved, and
         config_change.
+
     - name: type
       type: keyword
       description: The file type (file, dir, or symlink).
+
     - name: inode
       type: keyword
       description: The inode representing the file in the filesystem.
+
     - name: uid
       type: keyword
       description: The user ID (UID) of the file owner.
+
     - name: owner
       type: keyword
       description: The file owner's username.
+
     - name: gid
       type: keyword
       description: The primary group ID (GID) of the file.
+
     - name: group
       type: keyword
       description: The primary group name of the file.
+
     - name: sid
       type: keyword
       description: The security identifier (SID) of the file owner (Windows only).
+
     - name: mode
       type: keyword
       example: 0640
       description: The mode of the file in octal representation.
+
     - name: size
       type: long
       description: The file size in bytes (field is only added when `type` is `file`).
+
     - name: mtime
       type: date
       description: The last modified time of the file (time when content was modified).
+
     - name: ctime
       type: date
       description: The last change time of the file (time when metadata was changed).
+
     - name: hashed
       type: boolean
       description: >
         Boolean indicating if the event includes any file hashes.
+
     - name: md5
       type: keyword
       description: MD5 hash of the file.
+
     - name: sha1
       type: keyword
       description: SHA1 hash of the file.
+
     - name: sha224
       type: keyword
       description: SHA224 hash of the file.
+
     - name: sha256
       type: keyword
       description: SHA256 hash of the file.
+
     - name: sha384
       type: keyword
       description: SHA384 hash of the file.
+
     - name: sha3_224
       type: keyword
       description: SHA3_224 hash of the file.
+
     - name: sha3_256
       type: keyword
       description: SHA3_256 hash of the file.
+
     - name: sha3_384
       type: keyword
       description: SHA3_384 hash of the file.
+
     - name: sha3_512
       type: keyword
       description: SHA3_512 hash of the file.
+
     - name: sha512
       type: keyword
       description: SHA512 hash of the file.
+
     - name: sha512_224
       type: keyword
       description: SHA512/224 hash of the file.
+
     - name: sha512_256
       type: keyword
       description: SHA512/256 hash of the file.

auditbeat/module/audit/file/event.go

@@ -91,7 +91,8 @@ func (t Type) String() string {
 
 // Enum of possible file.Types.
 const (
-	FileType Type = iota
+	UnknownType Type = iota // Typically seen in deleted notifications where the object is gone.
+	FileType
 	DirType
 	SymlinkType
 )
@@ -237,7 +238,7 @@ func buildMapStr(e *Event) common.MapStr {
 			m["size"] = info.Size
 		}
 
-		if info.Type != 0 {
+		if info.Type != UnknownType {
 			m["type"] = info.Type.String()
 		}
 

auditbeat/module/audit/file/event_test.go

@@ -20,6 +20,7 @@ func testEvent() *Event {
 		Source:    SourceScan,
 		Action:    ConfigChange,
 		Info: &Metadata{
+			Type:  FileType,
 			Inode: 123,
 			UID:   500,
 			GID:   500,

auditbeat/module/audit/file/flatbuffers.go

@@ -104,6 +104,8 @@ func fbWriteMetadata(b *flatbuffers.Builder, m *Metadata) flatbuffers.UOffsetT {
 	}
 	schema.MetadataAddMode(b, uint32(m.Mode))
 	switch m.Type {
+	case UnknownType:
+		schema.MetadataAddType(b, schema.TypeUnknown)
 	case FileType:
 		schema.MetadataAddType(b, schema.TypeFile)
 
@@ -234,6 +236,8 @@ func fbDecodeMetadata(e *schema.Event) *Metadata {
 		rtn.Type = DirType
 	case schema.TypeSymlink:
 		rtn.Type = SymlinkType
+	default:
+		rtn.Type = UnknownType
 	}
 
 	return rtn

auditbeat/module/audit/file/flatbuffers_test.go

@@ -20,9 +20,9 @@ func TestFBEncodeDecode(t *testing.T) {
 		t.Fatal("decode returned nil")
 	}
 
-	assert.EqualValues(t, *e.Info, *out.Info)
+	assert.Equal(t, *e.Info, *out.Info)
 	e.Info, out.Info = nil, nil
-	assert.EqualValues(t, e, out)
+	assert.Equal(t, e, out)
 }
 
 func BenchmarkFBEncodeEvent(b *testing.B) {

auditbeat/module/audit/file/schema.fbs

@@ -16,6 +16,7 @@ enum Source : ubyte {
 }
 
 enum Type : ubyte {
+  Unknown,
   File,
   Dir,
   Symlink,
@@ -30,7 +31,7 @@ table Metadata {
   size:ulong;
   mtime_ns:long;
   ctime_ns:long;
-  type:Type;
+  type:Type = 1;
 }
 
 table Hash {

auditbeat/module/audit/file/schema/Metadata.go

@@ -123,7 +123,7 @@ func (rcv *Metadata) Type() byte {
 	if o != 0 {
 		return rcv._tab.GetByte(o + rcv._tab.Pos)
 	}
-	return 0
+	return 1
 }
 
 func (rcv *Metadata) MutateType(n byte) bool {
@@ -158,7 +158,7 @@ func MetadataAddCtimeNs(builder *flatbuffers.Builder, ctimeNs int64) {
 	builder.PrependInt64Slot(7, ctimeNs, 0)
 }
 func MetadataAddType(builder *flatbuffers.Builder, type_ byte) {
-	builder.PrependByteSlot(8, type_, 0)
+	builder.PrependByteSlot(8, type_, 1)
 }
 func MetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()

auditbeat/module/audit/file/schema/Type.go

@@ -3,12 +3,14 @@
 package schema
 
 const (
-	TypeFile    = 0
-	TypeDir     = 1
-	TypeSymlink = 2
+	TypeUnknown = 0
+	TypeFile    = 1
+	TypeDir     = 2
+	TypeSymlink = 3
 )
 
 var EnumNamesType = map[int]string{
+	TypeUnknown: "Unknown",
 	TypeFile:    "File",
 	TypeDir:     "Dir",
 	TypeSymlink: "Symlink",

libbeat/scripts/generate_fields_docs.py

@@ -62,6 +62,10 @@ def document_field(output, field, path):
         if not field["enable"]:
             output.write("{}\n\n".format("Object is not enabled."))
 
+    if "multi_fields" in field:
+        for subfield in field["multi_fields"]:
+            document_field(output, subfield, path + "." + subfield["name"])
+
 
 def fields_to_asciidoc(input, output, beat):