Merge pull request #172 from MannLabs/140-plot-history

140 plot history

tests/e2e_tests/calc_metrics.py

@@ -11,6 +11,9 @@
 import pandas as pd
 import neptune
 
+import matplotlib.pyplot as plt
+from datetime import datetime
+
 from tests.e2e_tests.prepare_test_data import get_test_case, OUTPUT_DIR_NAME
 
 NEPTUNE_PROJECT_NAME = os.environ.get("NEPTUNE_PROJECT_NAME")
@@ -118,6 +121,65 @@ def _calc(self):
             self._metrics[f"{self._name}/{col}_std"] = df[col].std()
 
 
+def _basic_plot(df: pd.DataFrame, test_case: str, metric: str, metric_std: str = None):
+    """Draw a basic line plot of `metric` for `test_case` over time."""
+
+    df = (
+        df[df["test_case"] == test_case]
+        .sort_index(ascending=False)
+        .reset_index(drop=True)
+    )
+
+    fig, ax = plt.subplots()
+    ax.scatter(x=df.index, y=df[metric])
+    if metric_std:
+        ax.errorbar(x=df.index, y=df[metric], yerr=df[metric_std])
+
+    ax.set_title(f"test_case: {test_case}, metric: {metric}")
+    ax.set_ylabel(metric)
+    ax.set_xlabel("test runs")
+
+    labels = []
+    for x, y, z in zip(
+        df["sys/creation_time"],
+        df["branch_name"],
+        df["short_sha"],
+    ):
+        fmt = "%Y-%m-%d %H:%M:%S.%f"
+        dt = datetime.strptime(str(x), fmt)
+        x = dt.strftime("%Y%m%d_%H:%M:%S")
+
+        labels.append(f"{x}:\n{y} [{z}]")
+
+    ax.set_xticks(df.index, labels, rotation=66)
+
+    return fig
+
+
+def _get_history_plots(test_results: dict, metrics_classes: list):
+    """Get all past runs from neptune, add the current one and create plots."""
+
+    test_results = test_results.copy()
+    test_results["sys/creation_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
+    test_results_df = pd.DataFrame(test_results, index=[0])
+
+    project = neptune.init_project(project=NEPTUNE_PROJECT_NAME, mode="read-only")
+    runs_table_df = project.fetch_runs_table().to_pandas()
+
+    df = pd.concat([runs_table_df, test_results_df])
+
+    test_case_name = test_results["test_case"]
+
+    figs = []
+    for metrics_class in [cls.__name__ for cls in metrics_classes]:
+        # TODO find a smarter way to get the metrics
+        for metric in [k for k in test_results.keys() if k.startswith(metrics_class)]:
+            fig = _basic_plot(df, test_case_name, metric)
+            figs.append((metric, fig))
+
+    return figs
+
+
 if __name__ == "__main__":
     test_case_name = sys.argv[1]
     run_time_minutes = int(sys.argv[2]) / 60
@@ -167,4 +229,12 @@ def _calc(self):
             if os.path.exists(file_path):
                 neptune_run["output/" + file_name].track_files(file_path)
 
+        try:
+            history_plots = _get_history_plots(test_results, metrics_classes)
+
+            for name, plot in history_plots:
+                neptune_run[f"plots/{name}"].upload(plot)
+        except Exception as e:
+            print(f"no plots today: {e}")
+
         neptune_run.stop()