Skip to content

Commit

Permalink
Allow to save the result of a tournament to a file
Browse files Browse the repository at this point in the history 
With this commit we allow users to save a tournament report by specifying the 
already known command line parameters `--report-format` and `--report-file`.

Closes #249
  • Loading branch information
@keyur9 @danielmitterdorfer
keyur9 authored and danielmitterdorfer committed Apr 4, 2017
1 parent 95f5348 commit fcf066d
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 55 deletions.
9 changes: 9 additions & 0 deletions esrally/rally.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,15 @@ def positive_number(v):
"--contender",
help="Race timestamp of the contender (see %s list races)" % PROGRAM_NAME,
default="")
compare_parser.add_argument(
"--report-format",
help="define the output format for the command line report (default: markdown).",
choices=["markdown", "csv"],
default="markdown")
compare_parser.add_argument(
"--report-file",
help="write the command line report also to the provided file",
default="")

config_parser = subparsers.add_parser("configure", help="Write the configuration file or reconfigure Rally")
for p in [parser, config_parser]:
Expand Down
123 changes: 68 additions & 55 deletions esrally/reporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,42 @@ def print_internal(message):
def print_header(message):
print_internal(console.format.bold(message))

def write_single_report(report_file, report_format, cwd, headers, data_plain, data_rich, write_header=True, show_also_in_console=True):

if report_format == "markdown":
formatter = format_as_markdown
elif report_format == "csv":
formatter = format_as_csv
else:
raise exceptions.SystemSetupError("Unknown report format '%s'" % report_format)

if show_also_in_console:
print_internal(formatter(headers, data_rich))
if len(report_file) > 0:
normalized_report_file = rio.normalize_path(report_file, cwd)
logger.info("Writing report to [%s] (user specified: [%s]) in format [%s]" %
(normalized_report_file, report_file, report_format))
# ensure that the parent folder already exists when we try to write the file...
rio.ensure_dir(rio.dirname(normalized_report_file))
with open(normalized_report_file, mode="a+", encoding="UTF-8") as f:
f.writelines(formatter(headers, data_plain, write_header))

def format_as_markdown(headers, data, write_header=True):
rendered = tabulate.tabulate(data, headers=headers, tablefmt="pipe", numalign="right", stralign="right")
if write_header:
return rendered + "\n"
else:
# remove all header data (it's not possible to do so entirely with tabulate directly...)
return "\n".join(rendered.splitlines()[2:]) + "\n"

def format_as_csv(headers, data, write_header=True):
with io.StringIO() as out:
writer = csv.writer(out)
if write_header:
writer.writerow(headers)
for metric_record in data:
writer.writerow(metric_record)
return out.getvalue()

class Stats:
def __init__(self, store, challenge, lap=None):
Expand Down Expand Up @@ -232,50 +268,12 @@ def report(self, t):

def write_report(self, metrics_table, meta_info_table):
report_file = self._config.opts("reporting", "output.path")

self.write_single_report(report_file, headers=["Lap", "Metric", "Operation", "Value", "Unit"], data=metrics_table,
write_header=self.needs_header())

if self.is_final_report() and len(report_file) > 0:
self.write_single_report("%s.meta" % report_file, headers=["Name", "Value"], data=meta_info_table, show_also_in_console=False)

def write_single_report(self, report_file, headers, data, write_header=True, show_also_in_console=True):
report_format = self._config.opts("reporting", "format")
if report_format == "markdown":
formatter = self.format_as_markdown
elif report_format == "csv":
formatter = self.format_as_csv
else:
raise exceptions.SystemSetupError("Unknown report format '%s'" % report_format)

if show_also_in_console:
print_internal(formatter(headers, data))
if len(report_file) > 0:
cwd = self._config.opts("node", "rally.cwd")
normalized_report_file = rio.normalize_path(report_file, cwd)
logger.info("Writing report to [%s] (user specified: [%s]) in format [%s]" %
(normalized_report_file, report_file, report_format))
# ensure that the parent folder already exists when we try to write the file...
rio.ensure_dir(rio.dirname(normalized_report_file))
with open(normalized_report_file, mode="a+", encoding="UTF-8") as f:
f.writelines(formatter(headers, data, write_header))

def format_as_markdown(self, headers, data, write_header=True):
rendered = tabulate.tabulate(data, headers=headers, tablefmt="pipe", numalign="right", stralign="right")
if write_header:
return rendered + "\n"
else:
# remove all header data (it's not possible to do so entirely with tabulate directly...)
return "\n".join(rendered.splitlines()[2:]) + "\n"

def format_as_csv(self, headers, data, write_header=True):
with io.StringIO() as out:
writer = csv.writer(out)
if write_header:
writer.writerow(headers)
for metric_record in data:
writer.writerow(metric_record)
return out.getvalue()
cwd = self._config.opts("node", "rally.cwd")
write_single_report(report_file, report_format, cwd, headers=["Lap", "Metric", "Operation", "Value", "Unit"], data_plain=metrics_table,
data_rich = metrics_table, write_header=self.needs_header())
if self.is_final_report() and len(report_file) > 0:
write_single_report("%s.meta" % report_file, report_format, cwd, headers=["Name", "Value"], data_plain = meta_info_table, data_rich = meta_info_table, show_also_in_console=False)

def report_throughput(self, stats, operation):
min, median, max, unit = stats.op_metrics[operation.name]["throughput"]
Expand Down Expand Up @@ -383,7 +381,7 @@ def report_meta_info(self):
class ComparisonReporter:
def __init__(self, config):
self._config = config

def report(self, r1, r2):
logger.info("Generating comparison report for baseline (invocation=[%s], track=[%s], challenge=[%s], car=[%s]) and "
"contender (invocation=[%s], track=[%s], challenge=[%s], car=[%s])" %
Expand Down Expand Up @@ -418,21 +416,18 @@ def report(self, r1, r2):
print_header("------------------------------------------------------")
print_internal("")

print_internal(self.format_as_table(self.metrics_table(baseline_stats, contender_stats)))
metric_table_plain = self.metrics_table(baseline_stats, contender_stats, plain = True)
metric_table_rich = self.metrics_table(baseline_stats, contender_stats, plain = False)
# Writes metric_table_rich to console, writes metric_table_plain to file
self.write_report(metric_table_plain,metric_table_rich)

def format_as_table(self, table):
return tabulate.tabulate(table,
headers=["Metric", "Operation", "Baseline", "Contender", "Diff", "Unit"],
tablefmt="pipe", numalign="right", stralign="right")

def metrics_table(self, baseline_stats, contender_stats):
def metrics_table(self, baseline_stats, contender_stats, plain):
self.plain=plain
metrics_table = []
metrics_table += self.report_total_times(baseline_stats, contender_stats)
metrics_table += self.report_merge_part_times(baseline_stats, contender_stats)

# metrics_table += self.report_cpu_usage(baseline_stats, contender_stats)
metrics_table += self.report_gc_times(baseline_stats, contender_stats)

metrics_table += self.report_disk_usage(baseline_stats, contender_stats)
metrics_table += self.report_segment_memory(baseline_stats, contender_stats)
metrics_table += self.report_segment_counts(baseline_stats, contender_stats)
Expand All @@ -445,6 +440,18 @@ def metrics_table(self, baseline_stats, contender_stats):
metrics_table += self.report_error_rate(baseline_stats, contender_stats, op)
return metrics_table

def format_as_table(self, table):
return tabulate.tabulate(table,
headers=["Metric", "Operation", "Baseline", "Contender", "Diff", "Unit"],
tablefmt="pipe", numalign="right", stralign="right")

def write_report(self, metrics_table, metrics_table_console):
report_file = self._config.opts("reporting", "output.path")
report_format = self._config.opts("reporting", "format")
cwd = self._config.opts("node", "rally.cwd")
write_single_report(report_file, report_format, cwd, headers=["Metric", "Operation", "Baseline", "Contender", "Diff", "Unit"],
data_plain= metrics_table, data_rich= metrics_table_console, write_header=True)

def report_throughput(self, baseline_stats, contender_stats, operation):
b_min, b_median, b_max, b_unit = baseline_stats.op_metrics[operation]["throughput"]
c_min, c_median, c_max, c_unit = contender_stats.op_metrics[operation]["throughput"]
Expand Down Expand Up @@ -601,17 +608,23 @@ def line(self, metric, baseline, contender, operation, unit, treat_increase_as_i

def diff(self, baseline, contender, treat_increase_as_improvement, formatter=lambda x: x):
diff = formatter(contender - baseline)
if treat_increase_as_improvement:
if self.plain:
color_greater = lambda x: x
color_smaller = lambda x: x
color_neutral = lambda x: x
elif treat_increase_as_improvement:
color_greater = console.format.green
color_smaller = console.format.red
color_neutral = console.format.neutral
else:
color_greater = console.format.red
color_smaller = console.format.green
color_neutral = console.format.neutral

if diff > 0:
return color_greater("+%.5f" % diff)
elif diff < 0:
return color_smaller("%.5f" % diff)
else:
# tabulate needs this to align all values correctly
return console.format.neutral("%.5f" % diff)
return color_neutral("%.5f" % diff)

0 comments on commit fcf066d

Please sign in to comment.