docs: add dynamic entity-relationship diagram to docs

.github/workflows/dependency-review.yml

@@ -23,7 +23,22 @@ jobs:
           # compatible/incompatible licenses addressed here: https://www.apache.org/legal/resolved.html
           # find SPDX identifiers here: https://spdx.org/licenses/
           deny-licenses: MS-LPL, BUSL-1.1, QPL-1.0, Sleepycat, SSPL-1.0, CPOL-1.02, AGPL-3.0, GPL-1.0+, BSD-4-Clause-UC, NPL-1.0, NPL-1.1, JSON
-          # adding an exception for an ambigious license on store2, which has been resolved in the latest version. It's MIT: https://github.com/nbubna/store/blob/master/LICENSE-MIT
-          # adding exception for all applitools modules (eyes-cypress and its dependencies), which has an explicit OSS license approved by ASF
-          #    license: https://applitools.com/legal/open-source-terms-of-use/
-          allow-dependencies-licenses: 'pkg:npm/[email protected], pkg:npm/applitools/core, pkg:npm/applitools/core-base, pkg:npm/applitools/css-tree, pkg:npm/applitools/ec-client, pkg:npm/applitools/eg-socks5-proxy-server, pkg:npm/applitools/eyes, pkg:npm/applitools/eyes-cypress, pkg:npm/applitools/nml-client, pkg:npm/applitools/tunnel-client, pkg:npm/applitools/utils'
+          allow-dependencies-licenses:
+            # adding an exception for an ambigious license on store2, which has been resolved in
+            # the latest version. It's MIT: https://github.com/nbubna/store/blob/master/LICENSE-MIT
+            - 'pkg:npm/[email protected]'
+            # adding exception for all applitools modules (eyes-cypress and its dependencies),
+            # which has an explicit OSS license approved by ASF
+            # license: https://applitools.com/legal/open-source-terms-of-use/
+            - 'pkg:npm/applitools/core'
+            - 'pkg:npm/applitools/core-base'
+            - 'pkg:npm/applitools/css-tree'
+            - 'pkg:npm/applitools/ec-client'
+            - 'pkg:npm/applitools/eg-socks5-proxy-server'
+            - 'pkg:npm/applitools/eyes'
+            - 'pkg:npm/applitools/eyes-cypress'
+            - 'pkg:npm/applitools/nml-client'
+            - 'pkg:npm/applitools/tunnel-client'
+            - 'pkg:npm/applitools/utils'
+            # Selecting BSD-3-Clause licensing terms for node-forge to ensure compatibility with Apache
+            - 'pkg:npm/[email protected]'

.github/workflows/superset-docs-deploy.yml

@@ -39,6 +39,13 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: '18'
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
+      - name: Compute Entity Relationship diagram (ERD)
+        run: |
+          python scripts/erd.py
+          curl -L http://sourceforge.net/projects/plantuml/files/1.2023.7/plantuml.1.2023.7.jar/download > ~/plantuml.jar
+          java -jar ~/plantuml.jar -v -tsvg -r -o "${{ github.workspace }}/docs/static/img/erd.svg" "${{ github.workspace }}/scripts/erd/erd.plantuml"
       - name: yarn install
         run: |
           yarn install --check-cache

.rat-excludes

@@ -66,3 +66,8 @@ google-big-query.svg
 google-sheets.svg
 postgresql.svg
 snowflake.svg
+
+# docs-related
+docs/docs/.htaccess*
+erd.plantuml
+erd.svg

docs/docs/contributing/erd.mdx

@@ -0,0 +1,11 @@
+import InteractiveSVG from '../../src/components/InteractiveERDSVG';
+
+# Entity-Relationship Diagram
+
+Here is our interactive ERD:
+
+<InteractiveSVG />
+
+<br />
+
+[Download the .svg](https://github.com/apache/superset/tree/master/docs/static/img/erd.svg)

docs/package.json

@@ -39,6 +39,7 @@
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "react-github-btn": "^1.4.0",
+    "react-svg-pan-zoom": "^3.12.1",
     "stream": "^0.0.2",
     "swagger-ui-react": "^4.1.3",
     "url-loader": "^4.1.1"

docs/src/components/InteractiveERDSVG.jsx

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React from 'react';
+import { UncontrolledReactSVGPanZoom } from 'react-svg-pan-zoom';
+import ErdSvg from '../../static/img/erd.svg';
+
+function InteractiveERDSVG() {
+  return (
+    <UncontrolledReactSVGPanZoom
+      width="100%"
+      height="800"
+      background="#003153"
+      tool="auto"
+    >
+      <svg>
+        <ErdSvg />
+      </svg>
+    </UncontrolledReactSVGPanZoom>
+  );
+}
+
+export default InteractiveERDSVG;

docs/yarn.lock

@@ -1994,7 +1994,7 @@
     "@docusaurus/theme-search-algolia" "2.4.3"
     "@docusaurus/types" "2.4.3"
 
-"@docusaurus/[email protected]", "react-loadable@npm:@docusaurus/[email protected]":
+"@docusaurus/[email protected]":
   version "5.5.2"
   resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz"
   integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
@@ -8223,6 +8223,15 @@ prop-types@^15.0.0, prop-types@^15.5.8, prop-types@^15.6.2, prop-types@^15.7.2:
     object-assign "^4.1.1"
     react-is "^16.8.1"
 
+prop-types@^15.8.1:
+  version "15.8.1"
+  resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
+  integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
+  dependencies:
+    loose-envify "^1.4.0"
+    object-assign "^4.1.1"
+    react-is "^16.13.1"
+
 property-information@^5.0.0, property-information@^5.3.0:
   version "5.6.0"
   resolved "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz"
@@ -8841,7 +8850,7 @@ react-inspector@^5.1.1:
     is-dom "^1.0.0"
     prop-types "^15.0.0"
 
-react-is@^16.6.0, react-is@^16.7.0, react-is@^16.8.1:
+react-is@^16.13.1, react-is@^16.6.0, react-is@^16.7.0, react-is@^16.8.1:
   version "16.13.1"
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
   integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==
@@ -8878,6 +8887,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1:
   dependencies:
     "@babel/runtime" "^7.10.3"
 
+"react-loadable@npm:@docusaurus/[email protected]":
+  version "5.5.2"
+  resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz"
+  integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
+  dependencies:
+    "@types/react" "*"
+    prop-types "^15.6.2"
+
 react-redux@^7.2.4:
   version "7.2.6"
   resolved "https://registry.npmjs.org/react-redux/-/react-redux-7.2.6.tgz"
@@ -8925,6 +8942,14 @@ [email protected], react-router@^5.3.3:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
+react-svg-pan-zoom@^3.12.1:
+  version "3.12.1"
+  resolved "https://registry.yarnpkg.com/react-svg-pan-zoom/-/react-svg-pan-zoom-3.12.1.tgz#971de6163fbad0d2a98d3ad7eb09bd1941564376"
+  integrity sha512-ug1LHCN5qed56C64xFypr/ClajuMFkig1OKvwJrIgGeSyHOjWM7XGgSgeP3IfHAkNw8QEc6a31ggZRpTijWYRw==
+  dependencies:
+    prop-types "^15.8.1"
+    transformation-matrix "^2.14.0"
+
 react-syntax-highlighter@^15.4.5:
   version "15.4.5"
   resolved "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.4.5.tgz"
@@ -10069,6 +10094,11 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
 
+transformation-matrix@^2.14.0:
+  version "2.16.1"
+  resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a"
+  integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA==
+
 traverse@~0.6.6:
   version "0.6.6"
   resolved "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz"

scripts/erd.py

@@ -0,0 +1,204 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+"""
+This module contains utilities to auto-generate an
+Entity-Relationship Diagram (ERD) from SQLAlchemy
+and onto a plantuml file.
+"""
+import os
+from collections import defaultdict
+from collections.abc import Iterable
+from typing import Any, Optional
+
+import click
+import jinja2
+from flask.cli import FlaskGroup, with_appcontext
+
+from superset import app, db
+
+GROUPINGS: dict[str, Iterable[str]] = {
+    "Core": [
+        "css_templates",
+        "dynamic_plugin",
+        "favstar",
+        "dashboards",
+        "slices",
+        "user_attribute",
+        "embedded_dashboards",
+        "annotation",
+        "annotation_layer",
+        "tag",
+        "tagged_object",
+    ],
+    "System": ["ssh_tunnels", "keyvalue", "cache_keys", "key_value", "logs"],
+    "Alerts & Reports": ["report_recipient", "report_execution_log", "report_schedule"],
+    "Inherited from Flask App Builder (FAB)": [
+        "ab_user",
+        "ab_permission",
+        "ab_permission_view",
+        "ab_view_menu",
+        "ab_role",
+        "ab_register_user",
+    ],
+    "SQL Lab": ["query", "saved_query", "tab_state", "table_schema"],
+    "Data Assets": [
+        "dbs",
+        "table_columns",
+        "sql_metrics",
+        "tables",
+        "row_level_security_filters",
+        "sl_tables",
+        "sl_datasets",
+        "sl_columns",
+        "database_user_oauth2_tokens",
+    ],
+}
+# Table name to group name mapping (reversing the above one for easy lookup)
+TABLE_TO_GROUP_MAP: dict[str, str] = {}
+for group, tables in GROUPINGS.items():
+    for table in tables:
+        TABLE_TO_GROUP_MAP[table] = group
+
+
+def introspect_sqla_model(mapper: Any, seen: set[str]) -> dict[str, Any]:
+    """
+    Introspects a SQLAlchemy model and returns a data structure that
+    can be pass to a jinja2 template for instance
+
+    Parameters:
+    -----------
+    mapper: SQLAlchemy model mapper
+    seen: set of model identifiers to avoid duplicates
+
+    Returns:
+    --------
+    Dict[str, Any]: data structure for jinja2 template
+    """
+    table_name = mapper.persist_selectable.name
+    model_info: dict[str, Any] = {
+        "class_name": mapper.class_.__name__,
+        "table_name": table_name,
+        "fields": [],
+        "relationships": [],
+    }
+    # Collect fields (columns) and their types
+    for column in mapper.columns:
+        field_info: dict[str, str] = {
+            "field_name": column.key,
+            "type": str(column.type),
+        }
+        model_info["fields"].append(field_info)
+
+    # Collect relationships and identify types
+    for attr, relationship in mapper.relationships.items():
+        related_table = relationship.mapper.persist_selectable.name
+        # Create a unique identifier for the relationship to avoid duplicates
+        relationship_id = "-".join(sorted([table_name, related_table]))
+
+        if relationship_id not in seen:
+            seen.add(relationship_id)
+            squiggle = "||--|{"
+            if relationship.direction.name == "MANYTOONE":
+                squiggle = "}|--||"
+
+            relationship_info: dict[str, str] = {
+                "relationship_name": attr,
+                "related_model": relationship.mapper.class_.__name__,
+                "type": relationship.direction.name,
+                "related_table": related_table,
+            }
+            # Identify many-to-many by checking for secondary table
+            if relationship.secondary is not None:
+                squiggle = "}|--|{"
+                relationship_info["type"] = "many-to-many"
+                relationship_info["secondary_table"] = relationship.secondary.name
+
+            relationship_info["squiggle"] = squiggle
+            model_info["relationships"].append(relationship_info)
+    return model_info
+
+
+def introspect_models() -> dict[str, list[dict[str, Any]]]:
+    """
+    Introspects SQLAlchemy models and returns a data structure that
+    can be pass to a jinja2 template for rendering an ERD.
+
+    Returns:
+    --------
+    Dict[str, List[Dict[str, Any]]]: data structure for jinja2 template
+    """
+    data: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    seen_models: set[str] = set()
+    for model in db.Model.registry.mappers:
+        group_name = (
+            TABLE_TO_GROUP_MAP.get(model.mapper.persist_selectable.name)
+            or "Other Models"
+        )
+        model_data = introspect_sqla_model(model, seen_models)
+        data[group_name].append(model_data)
+    return data
+
+
+def generate_erd(file_path: str) -> None:
+    """
+    Generates a PlantUML ERD of the models/database
+
+    Parameters:
+    -----------
+    file_path: str
+        File path to write the ERD to
+    """
+    data = introspect_models()
+    templates_path = os.path.join(os.path.dirname(__file__), "templates")
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path))
+
+    # Load the template
+    template = env.get_template("erd.plantuml.template")
+    rendered = template.render(data=data)
+    print(rendered)
+    with open(file_path, "w") as f:
+        f.write(rendered)
+
+
+@click.command()
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(dir_okay=False, writable=True),
+    help="File to write the ERD to",
+)
+def erd(output: Optional[str] = None) -> None:
+    """
+    Generates a PlantUML ERD of the models/database
+
+    Parameters:
+    -----------
+    output: str, optional
+        File to write the ERD to, defaults to erd.plantuml if not provided
+    """
+    output = output or "./erd.plantuml"
+
+    click.secho(f"Creating file at {output}...", fg="green")
+    from superset.app import create_app
+
+    app = create_app()
+    with app.app_context():
+        generate_erd(output)
+
+
+if __name__ == "__main__":
+    erd()