feat: plotly express

NOTICE

@@ -21,7 +21,7 @@
 # pyyaml: https://github.com/yaml/pyyaml/blob/master/LICENSE
 # jinja2: https://github.com/noirbizarre/jinja2/blob/master/LICENSE
 # tqdm: https://github.com/tqdm/tqdm/blob/master/LICENCE
-# matplotlib: https://github.com/matplotlib/matplotlib/blob/master/LICENSE/LICENSE
+# plotly: https://github.com/plotly/plotly.py/blob/master/LICENSE.txt
 # joblib: https://github.com/joblib/joblib/blob/master/LICENSE.txt
 # pybase64: https://github.com/mayeut/pybase64/blob/master/LICENSE
 # htmlmin: https://github.com/mankyd/htmlmin/blob/master/LICENSE

popmon/pipeline/report_pipelines.py

@@ -251,7 +251,7 @@ def __init__(
                 settings=settings,
             ),
             # generate report
-            ReportGenerator(read_key=sections_key, store_key=store_key),
+            ReportGenerator(read_key=sections_key, store_key=store_key, settings=settings),
         ]
         if (
             isinstance(settings.report_filepath, (str, Path))

popmon/visualization/__init__.py

@@ -28,13 +28,6 @@
     TrafficLightSectionGenerator,
 )
 
-# set matplotlib backend to batch mode when running in shell
-# need to do this *before* matplotlib.pyplot gets imported
-from ..visualization.backend import set_matplotlib_backend
-
-set_matplotlib_backend()
-
-
 __all__ = [
     "SectionGenerator",
     "HistogramSection",

popmon/visualization/alert_section_generator.py

@@ -157,7 +157,12 @@ def transform(
                 plots = [e for e in plots if len(e["plot"])]
 
             features_w_metrics.append(
-                {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])}
+                {
+                    "name": feature,
+                    "types": ["traffic_lights"],
+                    "plots": sorted(plots, key=lambda plot: plot["name"]),
+                    "layout": [""],
+                }
             )
 
         sections.append(

popmon/visualization/histogram_section.py

@@ -142,16 +142,34 @@ def transform(self, data_obj: dict, sections: Optional[list] = None):
             ]
             plots = parallel(_plot_histograms, args)
 
+            layouts = []
+            plot_types = []
             # filter out potential empty plots
             plots = [e for e in plots if len(e["plot"])]
             plots = sorted(plots, key=lambda plot: plot["name"])
-
+            if len(plots) > 0:
+                layouts.append(plots[0]["layout"])
+                plot_types.append("histogram")
             # filter out potential empty heatmap plots, then prepend them to the sorted histograms
-            hplots = [h for h in heatmaps if isinstance(h, dict) and len(h["plot"])]
-
+            hplots = []
+            for h in heatmaps:
+                if isinstance(h, dict):
+                    if len(h["plot"]):
+                        hplots.append(h)
+
+            if len(hplots) > 0:
+                layouts.append(hplots[0]["layout"])
+                plot_types.append("heatmap")
             plots = hplots + plots
-
-            features_w_metrics.append({"name": feature, "plots": plots})
+            # print(plot_types,layouts)
+            features_w_metrics.append(
+                {
+                    "name": feature,
+                    "types": plot_types,
+                    "plots": plots,
+                    "layout": layouts,
+                }
+            )
 
         sections.append(
             {
@@ -236,7 +254,14 @@ def _plot_histograms(feature, date, hc_list, hist_names, top_n, max_nbins=1000):
     else:
         plot = ""
 
-    return {"name": date, "description": "", "plot": plot}
+    return {
+        "name": date,
+        "type": "histogram",
+        "description": "",
+        "plot": plot["data"],
+        "layout": plot["layout"],
+    }
+
 
 
 def _plot_heatmap(
@@ -323,13 +348,15 @@ def _plot_heatmap(
         if isinstance(heatmaps, list):
             plot = [hist_lookup(heatmaps, hist_name) for hist_name in hist_names]
         elif isinstance(heatmaps, dict):
-            plot = [heatmaps["plot"]]
+            plot = [heatmaps]
 
         plots = [
             {
                 "name": hist_names_formatted[hist_name],
-                "description": descriptions[hist_name],
-                "plot": pl,
+                "type": "heatmap",
+                "description": "",
+                "plot": pl["plot"],
+                "layout": pl["layout"],
                 "full_width": True,
             }
             for pl, hist_name in zip(plot, hist_names)
@@ -366,4 +393,4 @@ def get_top_categories(entries_list, bins, top_n):
 def hist_lookup(plot, hist_name):
     for pl in plot:
         if pl["name"] == hist_name:
-            return pl["plot"]
+            return pl

popmon/visualization/overview_section.py

@@ -177,6 +177,7 @@ def _plot_metrics(
 
     return {
         "name": "Alert frequency per Feature",
+        "type": "alert",
         "description": "",
         "plot": plot,
         "full_width": True,

popmon/visualization/report_generator.py

@@ -33,15 +33,17 @@ class ReportGenerator(Module):
     _input_keys = ("read_key",)
     _output_keys = ("store_key",)
 
-    def __init__(self, read_key, store_key):
+    def __init__(self, read_key, store_key, online_report=False):
         """Initialize an instance of ReportGenerator.
 
         :param str read_key: key of input sections data to read from the datastore
         :param str store_key: key for storing the html report code in the datastore
+        :para bool online_report: if false (default), the plotly.js code is included in the html report, else the report takes js code from cdn server which requires internet connection
         """
         super().__init__()
         self.read_key = read_key
         self.store_key = store_key
+        self.online_report = online_report if online_report is not None else False
 
     def get_description(self):
         return "HTML Report"
@@ -51,7 +53,7 @@ def transform(self, sections: list) -> str:
         sections_html = ""
         for i, section_info in enumerate(sections):
             sections_html += templates_env(
-                filename="section.html", section_index=i, **section_info
+                filename="section.html", section_index=i, zip=zip, **section_info
             )
 
         # get HTML template for the final report, insert placeholder data and compress the code
@@ -60,5 +62,6 @@ def transform(self, sections: list) -> str:
                 filename="core.html",
                 generator=f"popmon {version}",
                 sections=sections_html,
+                online_report=self.online_report,
             )
         )

popmon/visualization/section_generator.py

@@ -193,8 +193,17 @@ def transform(
             # filter out potential empty plots (from skip empty plots)
             if self.skip_empty_plots:
                 plots = [e for e in plots if len(e["plot"])]
+
+            layouts = []
+            if len(plots) > 0:
+                layouts.append(plots[0]["layout"])
             features_w_metrics.append(
-                {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])}
+                {
+                    "name": feature,
+                    "types": ["barplot"],
+                    "plots": sorted(plots, key=lambda plot: plot["name"]),
+                    "layout": layouts,
+                }
             )
 
         sections.append(
@@ -248,4 +257,19 @@ def _plot_metric(
         skip_empty=skip_empty,
     )
 
-    return {"name": metric, "description": get_stat_description(metric), "plot": plot}
+    if not isinstance(plot, dict):
+        return {
+            "name": metric,
+            "type": "barplot",
+            "description": get_stat_description(metric),
+            "plot": plot,
+            "layout": plot,
+        }
+
+    return {
+        "name": metric,
+        "type": "barplot",
+        "description": get_stat_description(metric),
+        "plot": plot["data"],
+        "layout": plot["layout"],
+    }

popmon/visualization/templates/card.html

@@ -1,4 +1,4 @@
-<div class="col-md-{% if 'full_width' in metric %}12{% else %}6{% endif %} mb-5">
+<div class="col-md-{% if 'full_width' in metric %}12{% else %}6{% endif %} mb-5" >
     <a name="{% if feature%}{{ feature.name }}-{%endif%}{{ metric.name }}"></a>
     <div class="card shadow-sm">
         <div class="card-body" style="text-align: center">
@@ -7,10 +7,26 @@ <h4 class="card-title">{{metric.name | fmt_metric}}</h4>
                 <p class="card-text">{{metric.description}}</p>
             {% endif %}
         </div>
-        {% if 'table' in metric.plot %}
+        <div id= "{% if feature%}{{ feature.name }}-{%endif%}{{ metric.name }}"> </div>
+        {% if (('Overview' in section_title) or ('traffic_light' in metric.type)) %}
             {{ metric.plot }}
         {% else %}
-        <img class="card-img-top" src="data:image/png;base64,{{metric.plot}}" alt="" />
-        {% endif %}
+            <script>
+                var observer = new IntersectionObserver(function(entries) {
+                    if(entries[0].isIntersecting === true){
+                        console.log("plotting graph");
+                        {% if 'barplot' in metric.type %}
+                            TESTER = document.getElementById("{% if feature%}{{ feature.name }}-{%endif%}{{ metric.name }}");
+                            Plotly.newPlot(TESTER, JSON.parse('{{ metric.plot | tojson }}'), JSON.parse('{{ metric.layout | tojson }}'),{displaylogo: false, modeBarButtonsToRemove: ['lasso2d']});
+                        {% else %}
+                            TESTER = document.getElementById("{% if feature%}{{ feature.name }}-{%endif%}{{ metric.name }}");
+                            Plotly.newPlot(TESTER, JSON.parse('{{ metric.plot | tojson }}'), feature{{ section_index }}{{ curr }}_layout["{{ metric.type }}"],{displaylogo: false, modeBarButtonsToRemove: ['lasso2d']});
+                        {% endif %}
+                    }
+                }, { threshold: [0] });
+
+                observer.observe(document.getElementById("{% if feature%}{{ feature.name }}-{%endif%}{{ metric.name }}"));  
+            </script>
+        {% endif %} 
     </div>
 </div>