Remove logging.files.suffix option and always use datetime suffixes

CHANGELOG.next.asciidoc

@@ -27,6 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
+- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -1443,11 +1443,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. auditbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.reference.yml

@@ -2355,11 +2355,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. filebeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/tests/system/filebeat.py

@@ -36,7 +36,7 @@ def access_registry(self, name=None, data_path=None):
         return Registry(data_path, name)
 
     def log_access(self, file=None):
-        file = file if file else self.beat_name + ".log"
+        file = file if file else self.beat_name + "-" + self.today + ".ndjson"
         return LogState(os.path.join(self.working_dir, file))
 
     def has_registry(self, name=None, data_path=None):

filebeat/tests/system/test_harvester.py

@@ -493,11 +493,11 @@ def test_boms(self, fb_encoding, py_encoding, bom):
         filebeat = self.start_beat(output=fb_encoding + ".log")
 
         self.wait_until(
-            lambda: self.output_has(lines=1, output_file="output/" + fb_encoding),
+            lambda: self.output_has(lines=1, output_file="output/" + fb_encoding + "-" + self.today + ".ndjson"),
             max_timeout=10)
 
         # Verify that output does not contain bom
-        output = self.read_output_json(output_file="output/" + fb_encoding)
+        output = self.read_output_json(output_file="output/" + fb_encoding + "-" + self.today + ".ndjson")
         assert output[0]["message"] == message
 
         filebeat.kill_and_wait()

filebeat/tests/system/test_input.py

@@ -633,10 +633,10 @@ def test_restart_recursive_glob(self):
         with open(testfile_path, 'a') as testfile:
             testfile.write("entry2\n")
 
-        filebeat = self.start_beat(output="filebeat2.log")
+        filebeat = self.start_beat()
 
         self.wait_until(
-            lambda: self.output_has_message("entry2"),
+            lambda: self.output_has_message("entry2", output_file="output/filebeat-"+self.today+"-1.ndjson"),
             max_timeout=10,
             name="output contains 'entry2'")
 
@@ -783,7 +783,7 @@ def test_inode_marker_based_identity_tracking_to_path_based(self):
         proc = self.start_beat()
 
         # on startup output is rotated
-        self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat.1"))
+        self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"))
         self.wait_until(lambda: self.output_has(lines=1))
         proc.check_kill_and_wait()
 

filebeat/tests/system/test_load.py

@@ -72,7 +72,7 @@ def test_no_missing_events(self):
 
         entry_list = []
 
-        with open(self.working_dir + "/output/filebeat") as f:
+        with open(self.working_dir + "/output/filebeat-" + self.today + ".ndjson") as f:
             for line in f:
                 content = json.loads(line)
                 v = int(content["message"])

filebeat/tests/system/test_multiline.py

@@ -148,11 +148,11 @@ def test_max_lines(self):
 
         # Checks line 3 is sent
         assert True == self.log_contains(
-            "MetaDataMappingService.java:388", "output/filebeat")
+            "MetaDataMappingService.java:388", "output/filebeat-" + self.today + ".ndjson")
 
         # Checks line 4 is not sent anymore
         assert False == self.log_contains(
-            "InternalClusterService.java:388", "output/filebeat")
+            "InternalClusterService.java:388", "output/filebeat-" + self.today + ".ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert 20 == len(output)
@@ -231,10 +231,10 @@ def test_max_bytes(self):
         output = self.read_output()
 
         # Check that first 60 chars are sent
-        assert True == self.log_contains("cluster.metadata", "output/filebeat")
+        assert True == self.log_contains("cluster.metadata", "output/filebeat-" + self.today + ".ndjson")
 
         # Checks that chars afterwards are not sent
-        assert False == self.log_contains("Zach", "output/filebeat")
+        assert False == self.log_contains("Zach", "output/filebeat-" + self.today + ".ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert 20 == len(output)

filebeat/tests/system/test_registrar.py

@@ -486,11 +486,11 @@ def test_restart_continue(self):
 
         # Output file was rotated
         self.wait_until(
-            lambda: self.output_has(lines=1, output_file="output/filebeat.1"),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + ".ndjson"),
             max_timeout=10)
 
         self.wait_until(
-            lambda: self.output_has(lines=1),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"),
             max_timeout=10)
 
         filebeat.check_kill_and_wait()
@@ -505,7 +505,7 @@ def test_restart_continue(self):
         # should never have been detected
         assert len(data) == 1
 
-        output = self.read_output()
+        output = self.read_output(output_file="output/filebeat-" + self.today + "-1.ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert len(output) == 1
@@ -592,15 +592,15 @@ def test_rotating_file_with_restart(self):
         with open(testfile_path, 'w') as testfile:
             testfile.write("entry3\n")
 
-        filebeat = self.start_beat(output="filebeat2.log")
+        filebeat = self.start_beat()
 
         # Output file was rotated
         self.wait_until(
-            lambda: self.output_has(lines=2, output_file="output/filebeat.1"),
+            lambda: self.output_has(lines=2),
             max_timeout=10)
 
         self.wait_until(
-            lambda: self.output_has(lines=1),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"),
             max_timeout=10)
 
         filebeat.check_kill_and_wait()
@@ -951,8 +951,8 @@ def test_restart_state(self):
             clean_inactive="3s",
         )
 
-        filebeat = self.start_beat(output="filebeat2.log")
-        logs = self.log_access("filebeat2.log")
+        filebeat = self.start_beat()
+        logs = self.log_access()
 
         # Write additional file
         for name in restart_files:

heartbeat/heartbeat.reference.yml

@@ -1589,11 +1589,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. heartbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

journalbeat/journalbeat.reference.yml

@@ -1386,11 +1386,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. journalbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Journalbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -67,8 +67,3 @@ logging.files:
   # Rotate existing logs on startup rather than appending to the existing
   # file. Defaults to true.
   # rotateonstartup: true
-
-  # Rotated files are either suffixed with a number e.g. {{.BeatName}}.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count