Add filestream stream IDs to k8s manifests. (#2788)

* Add filestream stream IDs to k8s manifests.

Each filestream data stream also needs a unique ID, in addition to the
input ID. The input ID is used by the agent itself, the stream level ID
is what is used in the Filebeat registry.

* Add filestream docs to the reference configuration.

* Describe what IDs are used for in cfg files.

* Document IDs in the k8s configuration.

_meta/config/common.p2.yml.tmpl

@@ -12,7 +12,7 @@ outputs:
 
 
 # Here you can configure your list of inputs. You can either configure all the inputs as a list of arrays
-# or create an "inputs.d" directory containing your input configurations. 
+# or create an "inputs.d" directory containing your input configurations.
 # See https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html for how to structure the "inputs.d" directory.
 inputs:
   # Collecting system metrics
@@ -38,16 +38,18 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
-  # Collecting log files
-  #- type: filestream
-  #  id: your-input-id
-  #  streams:
-  #    # Unique ID for this data stream used to track the state of the ingested files.
-  #    - id: your-filestream-stream-id
-  #      data_stream:
-  #        dataset: generic
-  #      paths:
-  #        - /var/log/*.log
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
 
 
 # agent.monitoring:

_meta/config/common.reference.p2.yml.tmpl

@@ -32,6 +32,19 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -330,6 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
+      # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
       - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
         type: filestream
         use_output: default
@@ -340,7 +341,10 @@ data:
         data_stream:
           namespace: default
         streams:
-          - data_stream:
+          # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+          # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+            data_stream:
               dataset: kubernetes.container_logs
               type: logs
             prospector.scanner.symlinks: true
@@ -360,7 +364,8 @@ data:
         type: filestream
         use_output: default
         streams:
-          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+          - id: hints-container-logs-${kubernetes.hints.container_id}
+            condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
             data_stream:
               dataset: kubernetes.hints.container_logs
               type: logs

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml

@@ -330,6 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
+      # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
       - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
         type: filestream
         use_output: default
@@ -340,7 +341,10 @@ data:
         data_stream:
           namespace: default
         streams:
-          - data_stream:
+          # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+          # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+            data_stream:
               dataset: kubernetes.container_logs
               type: logs
             prospector.scanner.symlinks: true
@@ -360,7 +364,8 @@ data:
         type: filestream
         use_output: default
         streams:
-          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+          - id: hints-container-logs-${kubernetes.hints.container_id}
+            condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
             data_stream:
               dataset: kubernetes.hints.container_logs
               type: logs

elastic-agent.reference.yml

@@ -38,6 +38,19 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #

elastic-agent.yml

@@ -18,7 +18,7 @@ outputs:
 
 
 # Here you can configure your list of inputs. You can either configure all the inputs as a list of arrays
-# or create an "inputs.d" directory containing your input configurations. 
+# or create an "inputs.d" directory containing your input configurations.
 # See https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html for how to structure the "inputs.d" directory.
 inputs:
   # Collecting system metrics
@@ -44,16 +44,18 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
-  # Collecting log files
-  #- type: filestream
-  #  id: your-input-id
-  #  streams:
-  #    # Unique ID for this data stream used to track the state of the ingested files.
-  #    - id: your-filestream-stream-id
-  #      data_stream:
-  #        dataset: generic
-  #      paths:
-  #        - /var/log/*.log
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
 
 
 # agent.monitoring: