Restructure client/server and process fields

Changed - client_ip - client.ip (changed to ip type) - client_port - client.port - client_server - client.domain - client_proc - client.process.name - client_cmdline - client.process.args (changed to an array) - direction - network.direction - ip - server.ip (changed to ip type) - port - server.port - server - server.domain - proc - server.process.name - cmdline - server.process.args (change to an array) Added - client.process.executable - client.process.working_directory - client.process.pid - client.process.ppid - client.process.start - server.process.executable - server.process.working_directory - server.process.pid - server.process.ppid - server.process.start The top-level `process` field is populated with the `server.process` or `client.process` (server takes priority). Likewise for flows `process` is populated with the `destination.process` or the `client.process` (destination takes priority).
elastic · Dec 12, 2018 · 0081e9f · 0081e9f
1 parent 66a9525
commit 0081e9f
Show file tree

Hide file tree

Showing 55 changed files with 1,016 additions and 791 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -28,6 +28,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 *Packetbeat*
 
 - Renamed the flow event fields to follow Elastic Common Schema. {pull}9121[9121]
+- Renamed several client and server fields. IP, port, and process metadata are
+  now contained under the client and server namespaces. {issue}9303[9303]
 
 *Winlogbeat*
 

diff --git a/libbeat/common/endpoint.go b/libbeat/common/endpoint.go
@@ -19,27 +19,28 @@ package common
 
 // Endpoint represents an endpoint in the communication.
 type Endpoint struct {
-	IP      string
-	Port    uint16
-	Name    string
-	Cmdline string
-	Proc    string
+	IP     string
+	Port   uint16
+	Domain string
+
+	// Process metadata.
+	Process
 }
 
 // MakeEndpointPair returns source and destination endpoints from a TCP or IP tuple
 // and a command-line tuple.
-func MakeEndpointPair(tuple BaseTuple, cmdlineTuple *CmdlineTuple) (src Endpoint, dst Endpoint) {
+func MakeEndpointPair(tuple BaseTuple, processTuple *ProcessTuple) (src Endpoint, dst Endpoint) {
 	src = Endpoint{
-		IP:      tuple.SrcIP.String(),
-		Port:    tuple.SrcPort,
-		Proc:    string(cmdlineTuple.Src),
-		Cmdline: string(cmdlineTuple.SrcCommand),
+		IP:   tuple.SrcIP.String(),
+		Port: tuple.SrcPort,
 	}
 	dst = Endpoint{
-		IP:      tuple.DstIP.String(),
-		Port:    tuple.DstPort,
-		Proc:    string(cmdlineTuple.Dst),
-		Cmdline: string(cmdlineTuple.DstCommand),
+		IP:   tuple.DstIP.String(),
+		Port: tuple.DstPort,
+	}
+	if processTuple != nil {
+		src.Process = processTuple.Src
+		dst.Process = processTuple.Dst
 	}
 	return src, dst
 }
diff --git a/libbeat/common/tuples.go b/libbeat/common/tuples.go
@@ -20,6 +20,7 @@ package common
 import (
 	"fmt"
 	"net"
+	"time"
 )
 
 // In order for the IpPortTuple and the TcpTuple to be used as
@@ -155,22 +156,31 @@ func (t *TCPTuple) Hashable() HashableTCPTuple {
 	return t.raw
 }
 
-// CmdlineTuple contains the source and destination process names, as found by
+// ProcessTuple contains the source and destination process names, as found by
 // the proc module.
-type CmdlineTuple struct {
-	// Source and destination processes names as specified in packetbeat.procs.monitored
-	Src, Dst []byte
-	// Source and destination full command lines
-	SrcCommand, DstCommand []byte
+type ProcessTuple struct {
+	Src, Dst Process
+}
+
+// Process contains process information.
+type Process struct {
+	PID       int       // Process ID.
+	PPID      int       // Parent process ID.
+	Name      string    // Name of process (or alias given by cmdline_grep config).
+	Args      []string  // Process arguments.
+	Exe       string    // Absolute path to exe.
+	CWD       string    // Current working directory.
+	StartTime time.Time // Start time of process.
 }
 
 // Reverse returns a copy of the receiver with the source and destination fields
 // swapped.
-func (c *CmdlineTuple) Reverse() CmdlineTuple {
-	return CmdlineTuple{
-		Src:        c.Dst,
-		Dst:        c.Src,
-		SrcCommand: c.DstCommand,
-		DstCommand: c.SrcCommand,
+func (c *ProcessTuple) Reverse() ProcessTuple {
+	if c == nil {
+		return ProcessTuple{}
+	}
+	return ProcessTuple{
+		Src: c.Dst,
+		Dst: c.Src,
 	}
 }
diff --git a/packetbeat/_meta/fields.common.yml b/packetbeat/_meta/fields.common.yml
@@ -4,29 +4,8 @@
       These fields contain data about the environment in which the
       transaction or flow was captured.
   fields:
-    - name: server
-      description: >
-        The name of the server that served the transaction.
-
-    - name: client_server
-      description: >
-        The name of the server that initiated the transaction.
-
-    - name: client_service
-      description: >
-        The name of the logical service that initiated the transaction.
-
-    - name: ip
-      description: >
-        The IP address of the server that served the transaction.
-      format: dotted notation.
-
-    - name: client_ip
-      description: >
-        The IP address of the server that initiated the transaction.
-      format: dotted notation.
-
     - name: real_ip
+      type: ip
       description: >
         If the server initiating the transaction is a proxy, this field
         contains the original client IP address.
@@ -37,24 +16,6 @@
         the `client_ip` for non proxy clients.
       format: Dotted notation.
 
-    - name: client_geoip
-      description: The GeoIP information of the client.
-      type: group
-      fields:
-        - name: location
-          type: geo_point
-          example: {lat: 51, lon: 9}
-          description: >
-            The GeoIP location of the `client_ip` address. This field is available
-            only if you define a
-            https://www.elastic.co/guide/en/elasticsearch/plugins/master/using-ingest-geoip.html[GeoIP Processor] as a pipeline in the
-            https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-geoip.html[Ingest GeoIP processor plugin] or using Logstash.
-
-    - name: client_port
-      description: >
-        The layer 4 port of the process that initiated the transaction.
-      format: dotted notation.
-
     - name: transport
       description: >
         The transport protocol used for the transaction. If not specified, then
@@ -66,31 +27,45 @@
         The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.
       required: true
 
-    - name: port
-      description: >
-        The layer 4 port of the process that served the transaction.
-      format: dotted notation.
-
-    - name: proc
+    - name: server.process.name
       description: >
         The name of the process that served the transaction.
 
-    - name: cmdline
+    - name: server.process.args
       description: >
         The command-line of the process that served the transaction.
 
-    - name: client_proc
+    - name: server.process.executable
+      description: >
+        Absolute path to the server process executable.
+
+    - name: server.process.working_directory
+      description: >
+        The working directory of the server process.
+
+    - name: server.process.start
+      description: >
+        The time the server process started.
+
+    - name: client.process.name
       description: >
         The name of the process that initiated the transaction.
 
-    - name: client_cmdline
+    - name: client.process.args
       description: >
         The command-line of the process that initiated the transaction.
 
-    - name: release
+    - name: client.process.executable
+      description: >
+        Absolute path to the client process executable.
+
+    - name: client.process.working_directory
+      description: >
+        The working directory of the client process.
+
+    - name: client.process.start
       description: >
-        The software release of the service serving the transaction.
-        This can be the commit id or a semantic version.
+        The time the client process started.
 
 - key: flows_event
   title: "Flow Event"
@@ -120,15 +95,6 @@
     These fields contain data about the transaction itself.
   fields:
 
-    - name: direction
-      required: true
-      description: >
-        Indicates whether the transaction is inbound (emitted by server)
-        or outbound (emitted by the client). Values can be in or out. No defaults.
-      possible_values:
-        - in
-        - out
-
     - name: status
       description: >
         The high level status of the transaction. The way to compute this

diff --git a/packetbeat/_meta/sample_outputs/cassandra.json b/packetbeat/_meta/sample_outputs/cassandra.json
@@ -1,53 +1,64 @@
 {
-    "@timestamp": "2016-08-24T16:21:07.817Z",
-    "beat": {
-        "hostname": "Medcl.local",
-        "name": "Medcl.local"
+    "@timestamp": "2016-06-28T09:16:17.891Z",
+    "@metadata": {
+        "beat": "packetbeat",
+        "type": "doc",
+        "version": "7.0.0"
+    },
+    "bytes_in": 110,
+    "server": {
+        "domain": "host.example.com",
+        "ip": "127.0.0.1",
+        "port": 9042
+    },
+    "bytes_out": 871,
+    "host": {
+        "name": "host.example.com"
     },
-    "bytes_in": 88,
-    "bytes_out": 215,
     "cassandra": {
         "request": {
+            "query": "SELECT * FROM system_schema.tables WHERE keyspace_name = 'mykeyspace' AND table_name = 'users'",
             "headers": {
+                "version": "4",
                 "flags": "Default",
-                "length": 79,
+                "stream": 52,
                 "op": "QUERY",
-                "stream": 23,
-                "version": "4"
-            },
-            "query": "SELECT * FROM system_schema.keyspaces WHERE keyspace_name = 'mykeyspace'"
+                "length": 101
+            }
         },
         "response": {
             "headers": {
+                "version": "4",
                 "flags": "Default",
-                "length": 206,
+                "stream": 52,
                 "op": "RESULT",
-                "stream": 23,
-                "version": "4"
+                "length": 862
             },
             "result": {
-                "result_type": "rows",
+                "type": "rows",
                 "rows": {
                     "meta": {
-                        "col_count": 3,
-                        "flags": "GlobalTableSpec",
+                        "col_count": 19,
                         "keyspace": "system_schema",
-                        "table": "keyspaces"
+                        "table": "tables",
+                        "flags": "GlobalTableSpec"
                     },
                     "num_rows": 879461
                 }
             }
         }
     },
-    "client_ip": "127.0.0.1",
-    "client_port": 52749,
-    "client_proc": "",
-    "client_server": "Medcl.local",
-    "ip": "127.0.0.1",
-    "port": 9042,
-    "proc": "",
-    "responsetime": 6,
-    "server": "Medcl.local",
     "status": "OK",
+    "responsetime": 5,
+    "client": {
+        "port": 52749,
+        "domain": "host.example.com",
+        "ip": "127.0.0.1"
+    },
+    "agent": {
+        "type": "packetbeat",
+        "hostname": "host.example.com",
+        "version": "7.0.0"
+    },
     "type": "cassandra"
 }