[8.8](backport #2788) Add filestream stream IDs to k8s manifests. (#2855

) * Add filestream stream IDs to k8s manifests. (#2788) * Add filestream stream IDs to k8s manifests. Each filestream data stream also needs a unique ID, in addition to the input ID. The input ID is used by the agent itself, the stream level ID is what is used in the Filebeat registry. * Add filestream docs to the reference configuration. * Describe what IDs are used for in cfg files. * Document IDs in the k8s configuration. (cherry picked from commit 6a6720e) # Conflicts: # _meta/config/common.p2.yml.tmpl # elastic-agent.yml * Resolve conflict. --------- Co-authored-by: Craig MacKenzie <[email protected]>
elastic · Jun 13, 2023 · ba049e1 · ba049e1
1 parent 06cbfb1
commit ba049e1
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 4 deletions.
diff --git a/_meta/config/common.p2.yml.tmpl b/_meta/config/common.p2.yml.tmpl
@@ -9,6 +9,11 @@ outputs:
     # username: "elastic"
     # password: "changeme"
 
+
+
+# Here you can configure your list of inputs. You can either configure all the inputs as a list of arrays
+# or create an "inputs.d" directory containing your input configurations.
+# See https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html for how to structure the "inputs.d" directory.
 inputs:
   - type: system/metrics
     # Each input must have a unique ID.
@@ -32,6 +37,20 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
+
 # agent.monitoring:
 #   # enabled turns on monitoring of running processes
 #   enabled: true

diff --git a/_meta/config/common.reference.p2.yml.tmpl b/_meta/config/common.reference.p2.yml.tmpl
@@ -32,6 +32,19 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #

diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -330,6 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
+      # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
       - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
         type: filestream
         use_output: default
@@ -340,7 +341,10 @@ data:
         data_stream:
           namespace: default
         streams:
-          - data_stream:
+          # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+          # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+            data_stream:
               dataset: kubernetes.container_logs
               type: logs
             prospector.scanner.symlinks: true
@@ -360,7 +364,8 @@ data:
         type: filestream
         use_output: default
         streams:
-          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+          - id: hints-container-logs-${kubernetes.hints.container_id}
+            condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
             data_stream:
               dataset: kubernetes.hints.container_logs
               type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
@@ -330,6 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
+      # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
       - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
         type: filestream
         use_output: default
@@ -340,7 +341,10 @@ data:
         data_stream:
           namespace: default
         streams:
-          - data_stream:
+          # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+          # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+            data_stream:
               dataset: kubernetes.container_logs
               type: logs
             prospector.scanner.symlinks: true
@@ -360,7 +364,8 @@ data:
         type: filestream
         use_output: default
         streams:
-          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+          - id: hints-container-logs-${kubernetes.hints.container_id}
+            condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
             data_stream:
               dataset: kubernetes.hints.container_logs
               type: logs

diff --git a/elastic-agent.reference.yml b/elastic-agent.reference.yml
@@ -38,6 +38,19 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #

diff --git a/elastic-agent.yml b/elastic-agent.yml
@@ -15,6 +15,11 @@ outputs:
     # username: "elastic"
     # password: "changeme"
 
+
+
+# Here you can configure your list of inputs. You can either configure all the inputs as a list of arrays
+# or create an "inputs.d" directory containing your input configurations.
+# See https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html for how to structure the "inputs.d" directory.
 inputs:
   - type: system/metrics
     # Each input must have a unique ID.
@@ -38,6 +43,20 @@ inputs:
         - filesystem
         data_stream.dataset: system.filesystem
 
+#   # Collecting log files
+#   - type: filestream
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
+#       # Each filestream data stream creates a separate instance of the Filebeat filestream input.
+#       - id: your-filestream-stream-id
+#         data_stream:
+#           dataset: generic
+#         paths:
+#           - /var/log/*.log
+
+
 # agent.monitoring:
 #   # enabled turns on monitoring of running processes
 #   enabled: true