Remove logging.files.suffix option and always use datetime suffixes (…

…elastic#28927)

## What does this PR do?

This PR contains several improvements to the output file log rotation:
* Removes suffix option as discussed offline
* Filenames end in extension name so users get a hint about the contents
* Removes `time.Sleep` from testing code
* Datetime format no longer contains hour and minutes. In case of conflict on rotation, an index is appended to the filename.

## Why is it important?

Previously, log file suffixes were configurable. Users had the option to either add the count or the datetime to the end of the log file. From now on, we only allow datetime based naming. Hence the configuration option `logging.files.suffix` is removed.

Example:
Filebeat is writing logs to `filebeat-20211111.ndjson` actively. Then a few minutes later it gets rotated, and the new active file becomes `filebeat-20211111-1.ndjson`.

This change should help with Beats not being able to rotate files on Windows.

CHANGELOG.next.asciidoc

@@ -27,6 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
+- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -1443,11 +1443,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. auditbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.reference.yml

@@ -2355,11 +2355,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. filebeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/tests/system/filebeat.py

@@ -36,7 +36,7 @@ def access_registry(self, name=None, data_path=None):
         return Registry(data_path, name)
 
     def log_access(self, file=None):
-        file = file if file else self.beat_name + ".log"
+        file = file if file else self.beat_name + "-" + self.today + ".ndjson"
         return LogState(os.path.join(self.working_dir, file))
 
     def has_registry(self, name=None, data_path=None):

filebeat/tests/system/test_harvester.py

@@ -493,11 +493,11 @@ def test_boms(self, fb_encoding, py_encoding, bom):
         filebeat = self.start_beat(output=fb_encoding + ".log")
 
         self.wait_until(
-            lambda: self.output_has(lines=1, output_file="output/" + fb_encoding),
+            lambda: self.output_has(lines=1, output_file="output/" + fb_encoding + "-" + self.today + ".ndjson"),
             max_timeout=10)
 
         # Verify that output does not contain bom
-        output = self.read_output_json(output_file="output/" + fb_encoding)
+        output = self.read_output_json(output_file="output/" + fb_encoding + "-" + self.today + ".ndjson")
         assert output[0]["message"] == message
 
         filebeat.kill_and_wait()

filebeat/tests/system/test_input.py

@@ -633,10 +633,10 @@ def test_restart_recursive_glob(self):
         with open(testfile_path, 'a') as testfile:
             testfile.write("entry2\n")
 
-        filebeat = self.start_beat(output="filebeat2.log")
+        filebeat = self.start_beat()
 
         self.wait_until(
-            lambda: self.output_has_message("entry2"),
+            lambda: self.output_has_message("entry2", output_file="output/filebeat-"+self.today+"-1.ndjson"),
             max_timeout=10,
             name="output contains 'entry2'")
 
@@ -783,7 +783,7 @@ def test_inode_marker_based_identity_tracking_to_path_based(self):
         proc = self.start_beat()
 
         # on startup output is rotated
-        self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat.1"))
+        self.wait_until(lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"))
         self.wait_until(lambda: self.output_has(lines=1))
         proc.check_kill_and_wait()
 

filebeat/tests/system/test_load.py

@@ -72,7 +72,7 @@ def test_no_missing_events(self):
 
         entry_list = []
 
-        with open(self.working_dir + "/output/filebeat") as f:
+        with open(self.working_dir + "/output/filebeat-" + self.today + ".ndjson") as f:
             for line in f:
                 content = json.loads(line)
                 v = int(content["message"])

filebeat/tests/system/test_multiline.py

@@ -148,11 +148,11 @@ def test_max_lines(self):
 
         # Checks line 3 is sent
         assert True == self.log_contains(
-            "MetaDataMappingService.java:388", "output/filebeat")
+            "MetaDataMappingService.java:388", "output/filebeat-" + self.today + ".ndjson")
 
         # Checks line 4 is not sent anymore
         assert False == self.log_contains(
-            "InternalClusterService.java:388", "output/filebeat")
+            "InternalClusterService.java:388", "output/filebeat-" + self.today + ".ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert 20 == len(output)
@@ -231,10 +231,10 @@ def test_max_bytes(self):
         output = self.read_output()
 
         # Check that first 60 chars are sent
-        assert True == self.log_contains("cluster.metadata", "output/filebeat")
+        assert True == self.log_contains("cluster.metadata", "output/filebeat-" + self.today + ".ndjson")
 
         # Checks that chars afterwards are not sent
-        assert False == self.log_contains("Zach", "output/filebeat")
+        assert False == self.log_contains("Zach", "output/filebeat-" + self.today + ".ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert 20 == len(output)

filebeat/tests/system/test_registrar.py

@@ -486,11 +486,11 @@ def test_restart_continue(self):
 
         # Output file was rotated
         self.wait_until(
-            lambda: self.output_has(lines=1, output_file="output/filebeat.1"),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + ".ndjson"),
             max_timeout=10)
 
         self.wait_until(
-            lambda: self.output_has(lines=1),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"),
             max_timeout=10)
 
         filebeat.check_kill_and_wait()
@@ -505,7 +505,7 @@ def test_restart_continue(self):
         # should never have been detected
         assert len(data) == 1
 
-        output = self.read_output()
+        output = self.read_output(output_file="output/filebeat-" + self.today + "-1.ndjson")
 
         # Check that output file has the same number of lines as the log file
         assert len(output) == 1
@@ -592,15 +592,15 @@ def test_rotating_file_with_restart(self):
         with open(testfile_path, 'w') as testfile:
             testfile.write("entry3\n")
 
-        filebeat = self.start_beat(output="filebeat2.log")
+        filebeat = self.start_beat()
 
         # Output file was rotated
         self.wait_until(
-            lambda: self.output_has(lines=2, output_file="output/filebeat.1"),
+            lambda: self.output_has(lines=2),
             max_timeout=10)
 
         self.wait_until(
-            lambda: self.output_has(lines=1),
+            lambda: self.output_has(lines=1, output_file="output/filebeat-" + self.today + "-1.ndjson"),
             max_timeout=10)
 
         filebeat.check_kill_and_wait()
@@ -951,8 +951,8 @@ def test_restart_state(self):
             clean_inactive="3s",
         )
 
-        filebeat = self.start_beat(output="filebeat2.log")
-        logs = self.log_access("filebeat2.log")
+        filebeat = self.start_beat()
+        logs = self.log_access()
 
         # Write additional file
         for name in restart_files:

heartbeat/heartbeat.reference.yml

@@ -1589,11 +1589,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. heartbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

journalbeat/journalbeat.reference.yml

@@ -1386,11 +1386,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. journalbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Journalbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -67,8 +67,3 @@ logging.files:
   # Rotate existing logs on startup rather than appending to the existing
   # file. Defaults to true.
   # rotateonstartup: true
-
-  # Rotated files are either suffixed with a number e.g. {{.BeatName}}.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count