Skip to content

Commit

Permalink
Ignore JSON logs for merge parts analysis
Browse files Browse the repository at this point in the history
With this commit we ignore any JSON log files when analyzing merge
parts. Because Elasticsearch writes both plain text and JSON log files
with elastic/elasticsearch#36833, we need to ignore one of them in order
to avoid double-counting. We choose to ignore the JSON files for now as
this is backwards-compatible.
  • Loading branch information
danielmitterdorfer committed Jan 29, 2019
1 parent 5ec3161 commit 0ae7ea2
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions esrally/mechanic/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -656,12 +656,14 @@ def on_benchmark_stop(self):
merge_times = {}
for log_file in os.listdir(self.node_log_dir):
log_path = "%s/%s" % (self.node_log_dir, log_file)
if not io.is_archive(log_file):
if io.is_archive(log_file):
self.logger.debug("Skipping archived logs in [%s].", log_path)
elif io.has_extension(log_file, ".json"):
self.logger.debug("Skipping JSON-formatted logs in [%s].", log_path)
else:
self.logger.debug("Analyzing merge times in [%s]", log_path)
with open(log_path, mode="rt", encoding="utf-8") as f:
self._extract_merge_times(f, merge_times)
else:
self.logger.debug("Skipping archived logs in [%s].", log_path)
if merge_times:
self._store_merge_times(merge_times)
self.logger.info("Finished analyzing merge times. Extracted [%s] different merge time components.", len(merge_times))
Expand Down

0 comments on commit 0ae7ea2

Please sign in to comment.