Cherry-pick #22134 to 7.x: [Packetbeat] Refactor packetbeat for use w…

…ith Elastic Agent (#22546)

* [Packetbeat] Refactor packetbeat for use with Elastic Agent (#22134)

* Refactor packetbeat to support agent-based configuration

* Add documentation changes and a Changelog entry

* Update reference template

* Fix funny merge

* Incorporate feedback

* use streams instead of inputs

* support multiple sniffers

* fix shutdown_timeout behavior

(cherry picked from commit 8c05a41)

* Fix up changelog

CHANGELOG.next.asciidoc

@@ -589,6 +589,7 @@ same journal. {pull}18467[18467]
 
 `host` metadata fields when processing network data from network tap or mirror
 port. {pull}19209[19209]
+- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134]
 - Change build process for x-pack distribution {pull}21979[21979]
 
 

libbeat/cfgfile/list.go

@@ -157,7 +157,9 @@ func (r *RunnerList) Has(hash uint64) bool {
 // HashConfig hashes a given common.Config
 func HashConfig(c *common.Config) (uint64, error) {
 	var config map[string]interface{}
-	c.Unpack(&config)
+	if err := c.Unpack(&config); err != nil {
+		return 0, err
+	}
 	return hashstructure.Hash(config, nil)
 }
 

packetbeat/_meta/config/beat.reference.yml.tmpl

@@ -63,6 +63,9 @@ packetbeat.flows:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where flow events are indexed.
+  #index: my-custom-flow-index
+
 {{header "Transaction protocols"}}
 
 packetbeat.protocols:
@@ -73,6 +76,9 @@ packetbeat.protocols:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-icmp-index
+
 - type: amqp
   # Enable AMQP monitoring. Default: true
   #enabled: true
@@ -113,6 +119,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-amqp-index
+
 - type: cassandra
   #Cassandra port for traffic monitoring.
   ports: [9042]
@@ -143,6 +152,9 @@ packetbeat.protocols:
   # This option indicates which Operator/Operators will be ignored.
   #ignored_ops: ["SUPPORTED","OPTIONS"]
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-cassandra-index
+
 - type: dhcpv4
   # Configure the DHCP for IPv4 ports.
   ports: [67, 68]
@@ -183,6 +195,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-dhcpv4-index
+
 - type: http
   # Enable HTTP monitoring. Default: true
   #enabled: true
@@ -257,6 +272,9 @@ packetbeat.protocols:
   # be trimmed to this size. Default is 10 MB.
   #max_message_size: 10485760
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-http-index
+
 - type: memcache
   # Enable memcache monitoring. Default: true
   #enabled: true
@@ -309,6 +327,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-memcache-index
+
 - type: mysql
   # Enable mysql monitoring. Default: true
   #enabled: true
@@ -332,6 +353,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-mysql-index
+
 - type: pgsql
   # Enable pgsql monitoring. Default: true
   #enabled: true
@@ -355,6 +379,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-pgsql-index
+
 - type: redis
   # Enable redis monitoring. Default: true
   #enabled: true
@@ -387,6 +414,9 @@ packetbeat.protocols:
   # large enough to allow for pipelining.
   #queue_max_messages: 20000
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-redis-index
+
 - type: thrift
   # Enable thrift monitoring. Default: true
   #enabled: true
@@ -445,6 +475,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-thrift-index
+
 - type: mongodb
   # Enable mongodb monitoring. Default: true
   #enabled: true
@@ -478,6 +511,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-mongodb-index
+
 - type: nfs
   # Enable NFS monitoring. Default: true
   #enabled: true
@@ -501,6 +537,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-nfs-index
+
 - type: tls
   # Enable TLS monitoring. Default: true
   #enabled: true
@@ -531,6 +570,9 @@ packetbeat.protocols:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-tls-index
+
 - type: sip
   # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports.
   ports: [5060]
@@ -544,6 +586,9 @@ packetbeat.protocols:
   # Preserve original contents in event.original
   keep_original: true
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-sip-index
+
 {{header "Monitored processes"}}
 
 # Packetbeat can enrich events with information about the process associated