Configuration for rejected variables and warnings

- Rejected variables can be disabled (no longer grey or strikethrough)
- Warnings in the overview are automatically collapsed if over 20 item (configurable), can also be disabled completely by setting the limit to 0.

(#281)

src/pandas_profiling/config_dark.yaml

@@ -86,10 +86,19 @@ n_obs_unique: 5
 n_extreme_obs: 5
 n_freq_table_max: 10
 
+# Configuration related to the samples area
 samples:
     head: 10
     tail: 10
 
+# Configuration related to the warning overview (top) and per variable warnings
+warnings:
+    # Set to zero to disable showing warnings
+    collapse_if_more: 20
+
+# Configuration related to the rejection of variables
+reject_variables: True
+
 # When in a Jupyter notebook
 notebook:
     iframe:

src/pandas_profiling/config_default.yaml

@@ -86,10 +86,19 @@ n_obs_unique: 5
 n_extreme_obs: 5
 n_freq_table_max: 10
 
+# Configuration related to the samples area
 samples:
     head: 10
     tail: 10
 
+# Configuration related to the warning overview (top) and per variable warnings
+warnings:
+    # Set to zero to disable showing warnings
+    collapse_if_more: 20
+
+# Configuration related to the rejection of variables
+reject_variables: True
+
 # When in a Jupyter notebook
 notebook:
     iframe:

src/pandas_profiling/config_minimal.yaml

@@ -86,10 +86,19 @@ n_obs_unique: 5
 n_extreme_obs: 5
 n_freq_table_max: 10
 
+# Configuration related to the samples area
 samples:
     head: 10
     tail: 10
 
+# Configuration related to the warning overview (top) and per variable warnings
+warnings:
+    # Set to zero to disable showing warnings
+    collapse_if_more: 20
+
+# Configuration related to the rejection of variables
+reject_variables: True
+
 # When in a Jupyter notebook
 notebook:
     iframe:

src/pandas_profiling/model/messages.py

@@ -73,6 +73,16 @@ def __init__(
         self.column_name = column_name
         self.anchor_id = hash(column_name)
 
+    def fmt(self):
+        # TODO: render in template
+        name = self.message_type.name.replace("_", " ")
+        if name == "HIGH CORRELATION":
+            name = '<abbr title="This variable has a high correlation with {num} fields: {title}">HIGH CORRELATION</abbr>'.format(
+                num=len(self.values["fields"]),
+                title=", ".join(self.values["fields"]),
+            )
+        return name
+
 
 def check_table_messages(table: dict) -> List[Message]:
     """Checks the overall dataset for warnings.

src/pandas_profiling/report/presentation/core/dataset.py

@@ -4,14 +4,15 @@
 
 
 class Dataset(ItemRenderer):
-    def __init__(self, package, date, values, messages, variables, **kwargs):
+    def __init__(self, package, date, values, messages, collapse_warnings, variables, **kwargs):
         super().__init__(
             "dataset",
             {
                 "date": date,
                 "values": values,
                 "messages": messages,
                 "variables": variables,
+                "collapse_warnings": collapse_warnings,
                 "package": package,
             },
             **kwargs

src/pandas_profiling/report/presentation/flavours/html/templates/overview/overview.html

@@ -55,7 +55,7 @@
         </a>
     </div>
 
-    <div id="reproduction" class="row collapse">
+    <div id="reproduction" class="row collapse" aria-expanded="false">
         <div class="col-sm-12">
             <p class="h2">Reproduction info</p>
             <table class="table table-condensed stats">
@@ -84,13 +84,13 @@
     {% if messages %}
     <div class="col-sm-12 text-right">
         <a role="button" data-toggle="collapse" data-target="#warnings"
-           aria-expanded="true" aria-controls="collapseExample" class="">
+           aria-expanded="{% if collapse_warnings %}false{% else %}true{% endif %}" aria-controls="collapseExample" class="">
             Toggle Warnings
         </a>
     </div>
 
-    <div id="warnings" class="row collapse in" aria-expanded="true">
-        <div class="col-sm-12" style="padding-left: 1em;">
+    <div id="warnings" class="row collapse{% if not collapse_warnings %} in{% endif %}" aria-expanded="{% if collapse_warnings %}false{% else %}true{% endif %}">
+        <div class="col-sm-12" style="padding-left: 1em;}">
             <p class="h2">Warnings</p>
             <table class="table table-condensed list-warnings">
                 {% for message in messages %}

src/pandas_profiling/report/presentation/flavours/html/templates/overview/warnings/warning_unsupported.html

@@ -2,5 +2,5 @@
     <a class="anchor" href="#pp_var_{{ message.anchor_id }}"><code>{{ message.column_name }}</code></a> is an unsupported type, check if it needs cleaning or further analysis
 </td>
 <td>
-    <span class="label label-warning">Warning</span>
+    <span class="label label-warning">Rejected</span>
 </td>

src/pandas_profiling/report/structure/report.py

@@ -1,6 +1,7 @@
 """Generate the report."""
 
 import pandas_profiling.visualisation.plot as plot
+from pandas_profiling.config import config
 from pandas_profiling.model.base import (
     Boolean,
     Real,
@@ -110,22 +111,13 @@ def render_variables_section(dataframe_summary: dict) -> list:
     templs = []
 
     for idx, summary in dataframe_summary["variables"].items():
-        # TODO: move to render
         # Common template variables
-        def fmt_warning(warning):
-            name = warning.message_type.name.replace("_", " ")
-            if name == "HIGH CORRELATION":
-                name = '<abbr title="This variable has a high correlation with {num} fields: {title}">HIGH CORRELATION</abbr>'.format(
-                    num=len(warning.values["fields"]),
-                    title=", ".join(warning.values["fields"]),
-                )
-            return name
-
         warnings = [
-            fmt_warning(warning)
+            warning.fmt()
             for warning in dataframe_summary["messages"]
             if warning.column_name == idx
         ]
+
         warn_fields = [
             field
             for warning in dataframe_summary["messages"]
@@ -144,13 +136,18 @@ def fmt_warning(warning):
         # Per type template variables
         template_variables.update(type_to_func[summary["type"]](template_variables))
 
+        if config['reject_variables'].get(bool):
+            ignore = "ignore" in template_variables
+        else:
+            ignore = False
+
         templs.append(
             Preview(
                 template_variables["top"],
                 template_variables["bottom"],
                 anchor_id=template_variables["varid"],
                 name=idx,
-                ignore="ignore" in template_variables,
+                ignore=ignore,
             )
         )
 
@@ -190,13 +187,20 @@ def get_report_structure(date, sample: dict, summary: dict) -> Renderable:
       The profile report in HTML format
     """
 
+    collapse_warnings = config['warnings']['collapse_if_more'].get(int)
+    if collapse_warnings == 0:
+        warnings = []
+    else:
+        warnings = summary["messages"]
+
     sections = Sequence(
         [
             Dataset(
                 package=summary["package"],
                 date=date,
                 values=summary["table"],
-                messages=summary["messages"],
+                messages=warnings,
+                collapse_warnings=len(warnings) > collapse_warnings,
                 variables=summary["variables"],
                 name="Overview",
                 anchor_id="overview",