Skip to content

Commit

Permalink
Ignore JSON logs for merge parts analysis
Browse files Browse the repository at this point in the history
With this commit we ignore any JSON log files when analyzing merge
parts. Because Elasticsearch writes both plain text and JSON log files
with elastic/elasticsearch#36833, we need to ignore one of them in order
to avoid double-counting. We choose to ignore the JSON files for now as
this is backwards-compatible.

Relates #637
Relates elastic/elasticsearch#36833
  • Loading branch information
danielmitterdorfer authored Jan 29, 2019
1 parent ef07c10 commit 5b85f08
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions esrally/mechanic/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,12 +653,14 @@ def on_benchmark_stop(self):
merge_times = {}
for log_file in os.listdir(self.node_log_dir):
log_path = "%s/%s" % (self.node_log_dir, log_file)
if not io.is_archive(log_file):
if io.is_archive(log_file):
self.logger.debug("Skipping archived logs in [%s].", log_path)
elif io.has_extension(log_file, ".json"):
self.logger.debug("Skipping JSON-formatted logs in [%s].", log_path)
else:
self.logger.debug("Analyzing merge times in [%s]", log_path)
with open(log_path, mode="rt", encoding="utf-8") as f:
self._extract_merge_times(f, merge_times)
else:
self.logger.debug("Skipping archived logs in [%s].", log_path)
if merge_times:
self._store_merge_times(merge_times)
self.logger.info("Finished analyzing merge times. Extracted [%s] different merge time components.", len(merge_times))
Expand Down

0 comments on commit 5b85f08

Please sign in to comment.