From 3edefad1dc84d439af65527e79265d2de40a8d1a Mon Sep 17 00:00:00 2001 From: Lars George Date: Fri, 22 Sep 2023 14:50:07 +0200 Subject: [PATCH 01/11] Enhance Assessment Dashboard (#268) --- .../assessment/queries/database_summary.sql | 29 +++++++++++++++++++ .../ucx/assessment/queries/total_tables.sql | 23 ++++++++------- .../queries/workspace_access_total_groups.sql | 8 ----- 3 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 src/databricks/labs/ucx/assessment/queries/database_summary.sql delete mode 100644 src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql new file mode 100644 index 0000000000..d4a48595e6 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -0,0 +1,29 @@ +-- viz type=table, name=Database Summary, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- widget title=Database Summary, col=0, row=11, size_x=3, size_y=10 +select + `database`, + SUM(IS_TABLE) AS Tables, + SUM(IS_VIEW) AS Views, + SUM(IS_DBFS_Root) AS `DBFS Root`, + SUM(IS_DELTA) AS `Delta Tables`, + CASE + WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Full Asset Replication" + WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Asset Replication" + ELSE "In Place Sync" + END AS Upgrade +FROM +( +SELECT + database, + name, + object_type, + UPPER(table_format) AS format, + location, + CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS IS_TABLE, + CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS IS_VIEW, + CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, + CASE WHEN format LIKE "delta" THEN 1 ELSE 0 END AS IS_DELTA +FROM hive_metastore.ucx.tables +) +GROUP BY `database` +ORDER BY `database` diff --git a/src/databricks/labs/ucx/assessment/queries/total_tables.sql b/src/databricks/labs/ucx/assessment/queries/total_tables.sql index 94074153fa..6e408850e1 100644 --- a/src/databricks/labs/ucx/assessment/queries/total_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/total_tables.sql @@ -1,11 +1,14 @@ --- viz type=table, name=Table Types, columns=count,managed_tables,managed_pct,external_tables,external_pct,views,view_pct --- widget title=Table Types, col=0, row=1, size_x=3, size_y=3 +-- viz type=table, name=Table Types, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- widget title=Table Types, col=0, row=1, size_x=6, size_y=10 SELECT - COUNT(*) AS count, - SUM(IF(object_type == "MANAGED", 1, 0)) AS managed_tables, - ROUND(100 * SUM(IF(object_type == "MANAGED", 1, 0)) / COUNT(*), 0) AS managed_pct, - SUM(IF(object_type == "EXTERNAL", 1, 0)) AS external_tables, - ROUND(100 * SUM(IF(object_type == "EXTERNAL", 1, 0)) / COUNT(*), 0) AS external_pct, - SUM(IF(object_type == "VIEW", 1, 0)) AS views, - ROUND(100 * SUM(IF(object_type == "VIEW", 1, 0)) / COUNT(*), 0) AS view_pct -FROM $inventory.tables \ No newline at end of file + `database`, + name, + object_type AS type, + upper(table_format) AS format, + CASE WHEN object_type IN ("MANAGED", "EXTERNAL") THEN "TABLE" ELSE "VIEW" END AS table_view, + CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)<>"/mnt" THEN "DBFS ROOT" + WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)="/mnt" THEN "DBFS MOUNT" + ELSE "EXTERNAL" END AS storage, + CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, + location +FROM hive_metastore.ucx.tables diff --git a/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql b/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql deleted file mode 100644 index 04518ae471..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql +++ /dev/null @@ -1,8 +0,0 @@ --- viz type=table, name=Workspace Groups per Object Type, columns=object_type,objects --- widget title=Object Types, col=0, row=3, size_x=1, size_y=12 -SELECT - object_type, - COUNT(DISTINCT object_id) objects -FROM $inventory.permissions -GROUP BY 1 -ORDER BY objects DESC \ No newline at end of file From 5c97fe28663c92783c7251a048e7401690c84abe Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 11:41:03 -0400 Subject: [PATCH 02/11] Added Dashboard testing --- .../{total_tables.sql => all_tables.sql} | 2 +- .../assessment/queries/database_summary.sql | 4 ++-- .../assessment/queries/external_locations.sql | 6 +++++ .../ucx/assessment/queries/mount_points.sql | 6 +++++ .../integration/framework/test_dashboards.py | 22 ++++++++++++++++++- 5 files changed, 36 insertions(+), 4 deletions(-) rename src/databricks/labs/ucx/assessment/queries/{total_tables.sql => all_tables.sql} (95%) create mode 100644 src/databricks/labs/ucx/assessment/queries/external_locations.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/mount_points.sql diff --git a/src/databricks/labs/ucx/assessment/queries/total_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql similarity index 95% rename from src/databricks/labs/ucx/assessment/queries/total_tables.sql rename to src/databricks/labs/ucx/assessment/queries/all_tables.sql index 6e408850e1..9613ab7fd0 100644 --- a/src/databricks/labs/ucx/assessment/queries/total_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -11,4 +11,4 @@ SELECT ELSE "EXTERNAL" END AS storage, CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, location -FROM hive_metastore.ucx.tables +FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index d4a48595e6..0392a3e416 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,6 +1,6 @@ -- viz type=table, name=Database Summary, columns=database,name,type,table_format,table_view,storage,is_delta,location -- widget title=Database Summary, col=0, row=11, size_x=3, size_y=10 -select +SELECT `database`, SUM(IS_TABLE) AS Tables, SUM(IS_VIEW) AS Views, @@ -23,7 +23,7 @@ SELECT CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS IS_VIEW, CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, CASE WHEN format LIKE "delta" THEN 1 ELSE 0 END AS IS_DELTA -FROM hive_metastore.ucx.tables +FROM $inventory.tables ) GROUP BY `database` ORDER BY `database` diff --git a/src/databricks/labs/ucx/assessment/queries/external_locations.sql b/src/databricks/labs/ucx/assessment/queries/external_locations.sql new file mode 100644 index 0000000000..a20efcebf3 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/external_locations.sql @@ -0,0 +1,6 @@ +-- viz type=table, name=External Locations, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- widget title=External Locations, col=3, row=11, size_x=3, size_y=5 +SELECT + location +FROM + $inventory.external_locations diff --git a/src/databricks/labs/ucx/assessment/queries/mount_points.sql b/src/databricks/labs/ucx/assessment/queries/mount_points.sql new file mode 100644 index 0000000000..374c8b0560 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/mount_points.sql @@ -0,0 +1,6 @@ +-- viz type=table, name=Mount Points, columns=name, source +-- widget title=Mount Points, col=3, row=16, size_x=3, size_y=5 +SELECT + name, source +FROM + $inventory.mounts diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index 9e12767faf..a887f8f76a 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -1,9 +1,12 @@ import os +from unittest.mock import Mock import pytest from databricks.sdk import WorkspaceClient from databricks.sdk.service.sql import AccessControl, ObjectTypePlural, PermissionLevel +from databricks.labs.ucx.framework.dashboards import DashboardFromFiles +from databricks.labs.ucx.install import Installer from databricks.labs.ucx.mixins.redash import ( DashboardWidgetsAPI, QueryVisualizationsExt, @@ -15,7 +18,7 @@ # logging.getLogger("databricks").setLevel("DEBUG") -def test_creating_widgets(ws: WorkspaceClient): +def test_creating_widgets(ws: WorkspaceClient, make_warehouse, make_schema): pytest.skip() dashboard_widgets_api = DashboardWidgetsAPI(ws.api_client) query_visualizations_api = QueryVisualizationsExt(ws.api_client) @@ -58,3 +61,20 @@ def test_creating_widgets(ws: WorkspaceClient): y = query_visualizations_api.create_table(query.id, "ABC Viz", [VizColumn(name="databaseName", title="DB")]) print(y) + + +def test_building_dashboard(ws): + def _replace_inventory_variable(text: str) -> str: + return text.replace("$inventory", f"hive_metastore.ucx") + installer = Installer(ws) + warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + dash = DashboardFromFiles( + Mock(), + local_folder=installer._find_project_root() , + remote_folder=f"{installer._install_folder}/queries", + name="UCX Assessment", + warehouse_id=warehouse_id, + query_text_callback=installer._replace_inventory_variable, + ) + dashboard = dash.create_dashboard() + assert dashboard is not None From 713a41b916b5afbee0bde6042d6fec1dfd77ea42 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 12:40:10 -0400 Subject: [PATCH 03/11] Added Dashboard testing and two more queries --- .../labs/ucx/assessment/queries/external_locations.sql | 2 +- tests/integration/framework/test_dashboards.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/external_locations.sql b/src/databricks/labs/ucx/assessment/queries/external_locations.sql index a20efcebf3..98ba542916 100644 --- a/src/databricks/labs/ucx/assessment/queries/external_locations.sql +++ b/src/databricks/labs/ucx/assessment/queries/external_locations.sql @@ -1,4 +1,4 @@ --- viz type=table, name=External Locations, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- viz type=table, name=External Locations, columns=location -- widget title=External Locations, col=3, row=11, size_x=3, size_y=5 SELECT location diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index a887f8f76a..8173626000 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -67,10 +67,11 @@ def test_building_dashboard(ws): def _replace_inventory_variable(text: str) -> str: return text.replace("$inventory", f"hive_metastore.ucx") installer = Installer(ws) + installer._configure() warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] dash = DashboardFromFiles( Mock(), - local_folder=installer._find_project_root() , + local_folder=installer._find_project_root(), remote_folder=f"{installer._install_folder}/queries", name="UCX Assessment", warehouse_id=warehouse_id, From 4fdbff3cef657e40eb88bf5709385a1cbd295ab0 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 13:31:22 -0400 Subject: [PATCH 04/11] Removed integration test and added queries. --- .../labs/ucx/assessment/queries/all_tables.sql | 4 ++-- .../ucx/assessment/queries/database_summary.sql | 12 ++++++------ .../assessment/queries/external_locations.sql | 4 ++-- .../ucx/assessment/queries/mount_points.sql | 6 +++--- tests/integration/framework/test_dashboards.py | 17 ----------------- 5 files changed, 13 insertions(+), 30 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 9613ab7fd0..0894b8bf7f 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Table Types, columns=database,name,type,table_format,table_view,storage,is_delta,location --- widget title=Table Types, col=0, row=1, size_x=6, size_y=10 +-- widget title=Table Types, col=0, row=0, size_x=6, size_y=6 SELECT `database`, name, @@ -11,4 +11,4 @@ SELECT ELSE "EXTERNAL" END AS storage, CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, location -FROM $inventory.tables +FROM hive_metastore.ucx.tables diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 0392a3e416..708fe5b85c 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Database Summary, columns=database,name,type,table_format,table_view,storage,is_delta,location --- widget title=Database Summary, col=0, row=11, size_x=3, size_y=10 +-- widget title=Database Summary, col=0, row=6, size_x=3, size_y=8 SELECT `database`, SUM(IS_TABLE) AS Tables, @@ -7,8 +7,8 @@ SELECT SUM(IS_DBFS_Root) AS `DBFS Root`, SUM(IS_DELTA) AS `Delta Tables`, CASE - WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Full Asset Replication" - WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Asset Replication" + WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Asset Replication Required" + WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Non Delta Assets" ELSE "In Place Sync" END AS Upgrade FROM @@ -21,9 +21,9 @@ SELECT location, CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS IS_TABLE, CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS IS_VIEW, - CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, - CASE WHEN format LIKE "delta" THEN 1 ELSE 0 END AS IS_DELTA -FROM $inventory.tables + CASE WHEN SUBSTRING(location,0,3)="dbfs" AND SUBSTRING(location,4,9)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, + CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS IS_DELTA +FROM hive_metastore.ucx.tables ) GROUP BY `database` ORDER BY `database` diff --git a/src/databricks/labs/ucx/assessment/queries/external_locations.sql b/src/databricks/labs/ucx/assessment/queries/external_locations.sql index 98ba542916..58151449b8 100644 --- a/src/databricks/labs/ucx/assessment/queries/external_locations.sql +++ b/src/databricks/labs/ucx/assessment/queries/external_locations.sql @@ -1,6 +1,6 @@ -- viz type=table, name=External Locations, columns=location --- widget title=External Locations, col=3, row=11, size_x=3, size_y=5 +-- widget title=External Locations, col=3, row=6, size_x=3, size_y=4 SELECT location FROM - $inventory.external_locations + hive_metastore.ucx.external_locations diff --git a/src/databricks/labs/ucx/assessment/queries/mount_points.sql b/src/databricks/labs/ucx/assessment/queries/mount_points.sql index 374c8b0560..04259ead34 100644 --- a/src/databricks/labs/ucx/assessment/queries/mount_points.sql +++ b/src/databricks/labs/ucx/assessment/queries/mount_points.sql @@ -1,6 +1,6 @@ --- viz type=table, name=Mount Points, columns=name, source --- widget title=Mount Points, col=3, row=16, size_x=3, size_y=5 +-- viz type=table, name=Mount Points, columns=name,source +-- widget title=Mount Points, col=3, row=10, size_x=3, size_y=4 SELECT name, source FROM - $inventory.mounts + hive_metastore.ucx.mounts diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index 8173626000..81bb2dc11b 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -62,20 +62,3 @@ def test_creating_widgets(ws: WorkspaceClient, make_warehouse, make_schema): y = query_visualizations_api.create_table(query.id, "ABC Viz", [VizColumn(name="databaseName", title="DB")]) print(y) - -def test_building_dashboard(ws): - def _replace_inventory_variable(text: str) -> str: - return text.replace("$inventory", f"hive_metastore.ucx") - installer = Installer(ws) - installer._configure() - warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] - dash = DashboardFromFiles( - Mock(), - local_folder=installer._find_project_root(), - remote_folder=f"{installer._install_folder}/queries", - name="UCX Assessment", - warehouse_id=warehouse_id, - query_text_callback=installer._replace_inventory_variable, - ) - dashboard = dash.create_dashboard() - assert dashboard is not None From 437f083bd00c71802d1001b0f69529b47fa6a2e0 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 14:08:31 -0400 Subject: [PATCH 05/11] Added Clusters and Jobs queries --- .../labs/ucx/assessment/queries/all_tables.sql | 2 +- .../labs/ucx/assessment/queries/clusters.sql | 9 +++++++++ .../labs/ucx/assessment/queries/database_summary.sql | 2 +- src/databricks/labs/ucx/assessment/queries/jobs.sql | 10 ++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 src/databricks/labs/ucx/assessment/queries/clusters.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/jobs.sql diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 0894b8bf7f..37809f6664 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -11,4 +11,4 @@ SELECT ELSE "EXTERNAL" END AS storage, CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, location -FROM hive_metastore.ucx.tables +FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql new file mode 100644 index 0000000000..3008932f60 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/clusters.sql @@ -0,0 +1,9 @@ +-- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures +-- widget title=Clusters, col=0, row=15, size_x=3, size_y=4 +SELECT + cluster_id, + cluster_name, + creator, + CASE WHEN success=1 THEN "Compatible" ELSE "Incompatible" END AS compatible, + failures +FROM $inventory.clusters \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 708fe5b85c..c3c9c6841e 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -23,7 +23,7 @@ SELECT CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS IS_VIEW, CASE WHEN SUBSTRING(location,0,3)="dbfs" AND SUBSTRING(location,4,9)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS IS_DELTA -FROM hive_metastore.ucx.tables +FROM $inventory.tables ) GROUP BY `database` ORDER BY `database` diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql new file mode 100644 index 0000000000..340fefa13f --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -0,0 +1,10 @@ +-- viz type=table, name=Jobs, columns=job id,job name,creator,compatible,failures +-- widget title=Jobs, col=3, row=10, size_x=3, size_y=4 +SELECT + job_id, + job_name, + creator, + CASE WHEN success=1 THEN "Compatible" else "Incompatible" END AS compatible, + failures +FROM $inventory.jobs +WHERE job_name not like '[UCX]%' From a61cd6c546b0e51d73d510d86d79c71d103f92f5 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 14:20:02 -0400 Subject: [PATCH 06/11] Added Clusters and Jobs queries --- .../labs/ucx/assessment/queries/database_summary.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index c3c9c6841e..14147c0fa5 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -19,10 +19,10 @@ SELECT object_type, UPPER(table_format) AS format, location, - CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS IS_TABLE, - CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS IS_VIEW, - CASE WHEN SUBSTRING(location,0,3)="dbfs" AND SUBSTRING(location,4,9)<>"/mnt" THEN 1 ELSE 0 END AS IS_DBFS_Root, - CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS IS_DELTA + CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS is_table, + CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS is_view, + CASE WHEN SUBSTRING(location,0,3)="dbfs" AND SUBSTRING(location,4,9)<>"/mnt" THEN 1 ELSE 0 END AS is_dbfs_root, + CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS is_delta FROM $inventory.tables ) GROUP BY `database` From 7030b354f0b2e973fb379f82e73800bae6600a01 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 14:36:57 -0400 Subject: [PATCH 07/11] Important Commit!!! --- src/databricks/labs/ucx/assessment/queries/jobs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql index 340fefa13f..205bc729f1 100644 --- a/src/databricks/labs/ucx/assessment/queries/jobs.sql +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -1,4 +1,4 @@ --- viz type=table, name=Jobs, columns=job id,job name,creator,compatible,failures +-- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures -- widget title=Jobs, col=3, row=10, size_x=3, size_y=4 SELECT job_id, From b0e469ccd409a69216352285a505b9b3f790871c Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 22 Sep 2023 15:35:28 -0400 Subject: [PATCH 08/11] Fixed a couple of report issues. --- src/databricks/labs/ucx/assessment/queries/all_tables.sql | 4 ++-- .../labs/ucx/assessment/queries/database_summary.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 37809f6664..24aeb83429 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -6,8 +6,8 @@ SELECT object_type AS type, upper(table_format) AS format, CASE WHEN object_type IN ("MANAGED", "EXTERNAL") THEN "TABLE" ELSE "VIEW" END AS table_view, - CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)<>"/mnt" THEN "DBFS ROOT" - WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,10)="/mnt" THEN "DBFS MOUNT" + CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)<>"/mnt" THEN "DBFS ROOT" + WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)="/mnt" THEN "DBFS MOUNT" ELSE "EXTERNAL" END AS storage, CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, location diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 14147c0fa5..7d0af81d01 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -21,7 +21,7 @@ SELECT location, CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS is_table, CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS is_view, - CASE WHEN SUBSTRING(location,0,3)="dbfs" AND SUBSTRING(location,4,9)<>"/mnt" THEN 1 ELSE 0 END AS is_dbfs_root, + CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)<>"/mnt" THEN 1 ELSE 0 END AS is_dbfs_root, CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS is_delta FROM $inventory.tables ) From 8564de830494ee5eb82c05091ef0545b02f510a8 Mon Sep 17 00:00:00 2001 From: Lars George Date: Sun, 24 Sep 2023 16:29:55 +0200 Subject: [PATCH 09/11] Fix viz columns and format SQL code. --- .../ucx/assessment/queries/all_tables.sql | 34 ++++++---- .../labs/ucx/assessment/queries/clusters.sql | 14 +++-- .../assessment/queries/database_summary.sql | 62 +++++++++++-------- .../assessment/queries/external_locations.sql | 6 +- .../labs/ucx/assessment/queries/jobs.sql | 14 +++-- .../ucx/assessment/queries/mount_points.sql | 7 +-- 6 files changed, 79 insertions(+), 58 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 24aeb83429..e3579caf4c 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -1,14 +1,24 @@ --- viz type=table, name=Table Types, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location -- widget title=Table Types, col=0, row=0, size_x=6, size_y=6 -SELECT - `database`, - name, - object_type AS type, - upper(table_format) AS format, - CASE WHEN object_type IN ("MANAGED", "EXTERNAL") THEN "TABLE" ELSE "VIEW" END AS table_view, - CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)<>"/mnt" THEN "DBFS ROOT" - WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)="/mnt" THEN "DBFS MOUNT" - ELSE "EXTERNAL" END AS storage, - CASE WHEN format LIKE "delta" THEN "Yes" ELSE "No" END AS is_delta, - location +SELECT `database`, + name, + object_type AS type, + UPPER(table_format) AS format, + CASE + WHEN object_type IN ("MANAGED", + "EXTERNAL") THEN "TABLE" + ELSE "VIEW" + END AS table_view, + CASE + WHEN STARTSWITH(location, "/dbfs/") + AND NOT STARTSWITH(location, "/dbfs/mnt") THEN "DBFS ROOT" + WHEN STARTSWITH(location, "/dbfs/") + AND STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT" + ELSE "EXTERNAL" + END AS storage, + CASE + WHEN format LIKE "delta" THEN "Yes" + ELSE "No" + END AS is_delta, + location FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql index 3008932f60..8e470ff909 100644 --- a/src/databricks/labs/ucx/assessment/queries/clusters.sql +++ b/src/databricks/labs/ucx/assessment/queries/clusters.sql @@ -1,9 +1,11 @@ -- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures -- widget title=Clusters, col=0, row=15, size_x=3, size_y=4 -SELECT - cluster_id, - cluster_name, - creator, - CASE WHEN success=1 THEN "Compatible" ELSE "Incompatible" END AS compatible, - failures +SELECT cluster_id, + cluster_name, + creator, + CASE + WHEN success=1 THEN "Compatible" + ELSE "Incompatible" + END AS compatible, + failures FROM $inventory.clusters \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 7d0af81d01..12c03fa195 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,29 +1,39 @@ --- viz type=table, name=Database Summary, columns=database,name,type,table_format,table_view,storage,is_delta,location +-- viz type=table, name=Database Summary, columns=database,tables,views,dbfs root,delta tables,upgrade -- widget title=Database Summary, col=0, row=6, size_x=3, size_y=8 -SELECT - `database`, - SUM(IS_TABLE) AS Tables, - SUM(IS_VIEW) AS Views, - SUM(IS_DBFS_Root) AS `DBFS Root`, - SUM(IS_DELTA) AS `Delta Tables`, - CASE - WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Asset Replication Required" - WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Non Delta Assets" - ELSE "In Place Sync" - END AS Upgrade +SELECT `database`, + SUM(IS_TABLE) AS TABLES, + SUM(IS_VIEW) AS VIEWS, + SUM(IS_DBFS_Root) AS `DBFS Root`, + SUM(IS_DELTA) AS `Delta Tables`, + CASE + WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Asset Replication Required" + WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Non Delta Assets" + ELSE "In Place Sync" + END AS Upgrade FROM -( -SELECT - database, - name, - object_type, - UPPER(table_format) AS format, - location, - CASE WHEN object_type IN ("MANAGED","EXTERNAL") THEN 1 ELSE 0 END AS is_table, - CASE WHEN object_type="VIEW" THEN 1 ELSE 0 END AS is_view, - CASE WHEN SUBSTRING(location,1,4)="dbfs" AND SUBSTRING(location,6,4)<>"/mnt" THEN 1 ELSE 0 END AS is_dbfs_root, - CASE WHEN upper(format) LIKE "DELTA" THEN 1 ELSE 0 END AS is_delta -FROM $inventory.tables -) + (SELECT DATABASE, + name, + object_type, + UPPER(table_format) AS format, + LOCATION, + CASE + WHEN object_type IN ("MANAGED", + "EXTERNAL") THEN 1 + ELSE 0 + END AS is_table, + CASE + WHEN object_type="VIEW" THEN 1 + ELSE 0 + END AS is_view, + CASE + WHEN STARTSWITH(location, "/dbfs/") + AND NOT STARTSWITH(location, "/dbfs/mnt") THEN 1 + ELSE 0 + END AS is_dbfs_root, + CASE + WHEN UPPER(format) LIKE "DELTA" THEN 1 + ELSE 0 + END AS is_delta + FROM $inventory.tables) GROUP BY `database` -ORDER BY `database` +ORDER BY `database` \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/external_locations.sql b/src/databricks/labs/ucx/assessment/queries/external_locations.sql index 58151449b8..4aaf82e09e 100644 --- a/src/databricks/labs/ucx/assessment/queries/external_locations.sql +++ b/src/databricks/labs/ucx/assessment/queries/external_locations.sql @@ -1,6 +1,4 @@ -- viz type=table, name=External Locations, columns=location -- widget title=External Locations, col=3, row=6, size_x=3, size_y=4 -SELECT - location -FROM - hive_metastore.ucx.external_locations +SELECT location +FROM hive_metastore.ucx.external_locations diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql index 205bc729f1..fa2e62b756 100644 --- a/src/databricks/labs/ucx/assessment/queries/jobs.sql +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -1,10 +1,12 @@ -- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures -- widget title=Jobs, col=3, row=10, size_x=3, size_y=4 -SELECT - job_id, - job_name, - creator, - CASE WHEN success=1 THEN "Compatible" else "Incompatible" END AS compatible, - failures +SELECT job_id, + job_name, + creator, + CASE + WHEN success=1 THEN "Compatible" + ELSE "Incompatible" + END AS compatible, + failures FROM $inventory.jobs WHERE job_name not like '[UCX]%' diff --git a/src/databricks/labs/ucx/assessment/queries/mount_points.sql b/src/databricks/labs/ucx/assessment/queries/mount_points.sql index 04259ead34..3ae7e45c55 100644 --- a/src/databricks/labs/ucx/assessment/queries/mount_points.sql +++ b/src/databricks/labs/ucx/assessment/queries/mount_points.sql @@ -1,6 +1,5 @@ -- viz type=table, name=Mount Points, columns=name,source -- widget title=Mount Points, col=3, row=10, size_x=3, size_y=4 -SELECT - name, source -FROM - hive_metastore.ucx.mounts +SELECT name, + source +FROM hive_metastore.ucx.mounts \ No newline at end of file From 948822188417d6b8db03f487d44d71e7190845c3 Mon Sep 17 00:00:00 2001 From: Lars George Date: Mon, 25 Sep 2023 11:40:29 +0200 Subject: [PATCH 10/11] Applied feedback and added counter viz support. --- .../ucx/assessment/queries/all_tables.sql | 11 +----- .../labs/ucx/assessment/queries/clusters.sql | 7 +--- .../assessment/queries/count_total_tables.sql | 4 ++ .../assessment/queries/database_summary.sql | 39 +++++++------------ .../labs/ucx/assessment/queries/jobs.sql | 7 +--- .../labs/ucx/framework/dashboards.py | 34 +++++++++++++++- .../integration/framework/test_dashboards.py | 4 -- 7 files changed, 56 insertions(+), 50 deletions(-) create mode 100644 src/databricks/labs/ucx/assessment/queries/count_total_tables.sql diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index e3579caf4c..492b90a058 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -4,11 +4,7 @@ SELECT `database`, name, object_type AS type, UPPER(table_format) AS format, - CASE - WHEN object_type IN ("MANAGED", - "EXTERNAL") THEN "TABLE" - ELSE "VIEW" - END AS table_view, + IF(object_type IN ("MANAGED", "EXTERNAL"), "TABLE", "VIEW") AS table_view, CASE WHEN STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt") THEN "DBFS ROOT" @@ -16,9 +12,6 @@ SELECT `database`, AND STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT" ELSE "EXTERNAL" END AS storage, - CASE - WHEN format LIKE "delta" THEN "Yes" - ELSE "No" - END AS is_delta, + IF(format = "delta", "Yes", "No") AS is_delta, location FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql index 8e470ff909..56baee5bab 100644 --- a/src/databricks/labs/ucx/assessment/queries/clusters.sql +++ b/src/databricks/labs/ucx/assessment/queries/clusters.sql @@ -1,11 +1,8 @@ -- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures --- widget title=Clusters, col=0, row=15, size_x=3, size_y=4 +-- widget title=Clusters, col=0, row=15, size_x=3, size_y=8 SELECT cluster_id, cluster_name, creator, - CASE - WHEN success=1 THEN "Compatible" - ELSE "Incompatible" - END AS compatible, + IF(success = 1, "Compatible", "Incompatible") AS compatible, failures FROM $inventory.clusters \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql new file mode 100644 index 0000000000..2989f0e486 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql @@ -0,0 +1,4 @@ +-- viz type=counter, name=Total Table Count, counter_label=Total Tables, value_column=count_total_tables +-- widget title=Total Table Count, col=0, row=0, size_x=1, size_y=3 +SELECT count(*) AS count_total_tables +FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 12c03fa195..df089863d4 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,39 +1,26 @@ --- viz type=table, name=Database Summary, columns=database,tables,views,dbfs root,delta tables,upgrade +-- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgrade -- widget title=Database Summary, col=0, row=6, size_x=3, size_y=8 SELECT `database`, - SUM(IS_TABLE) AS TABLES, - SUM(IS_VIEW) AS VIEWS, - SUM(IS_DBFS_Root) AS `DBFS Root`, - SUM(IS_DELTA) AS `Delta Tables`, + SUM(is_table) AS tables, + SUM(is_view) AS views, + SUM(is_dbfs_root) AS dbfs_root, + SUM(is_delta) AS delta_tables, CASE - WHEN (SUM(IS_DBFS_Root)/SUM(IS_TABLE) > .3) THEN "Asset Replication Required" - WHEN (SUM(IS_DELTA)/SUM(IS_TABLE) < .7) THEN "Some Non Delta Assets" + WHEN (SUM(is_dbfs_root)/SUM(is_table) > .3) THEN "Asset Replication Required" + WHEN (SUM(is_delta)/SUM(is_table) < .7) THEN "Some Non Delta Assets" ELSE "In Place Sync" - END AS Upgrade + END AS upgrade FROM (SELECT DATABASE, name, object_type, UPPER(table_format) AS format, LOCATION, - CASE - WHEN object_type IN ("MANAGED", - "EXTERNAL") THEN 1 - ELSE 0 - END AS is_table, - CASE - WHEN object_type="VIEW" THEN 1 - ELSE 0 - END AS is_view, - CASE - WHEN STARTSWITH(location, "/dbfs/") - AND NOT STARTSWITH(location, "/dbfs/mnt") THEN 1 - ELSE 0 - END AS is_dbfs_root, - CASE - WHEN UPPER(format) LIKE "DELTA" THEN 1 - ELSE 0 - END AS is_delta + IF(object_type IN ("MANAGED", "EXTERNAL"), 1, 0) AS is_table, + IF(object_type = "VIEW", 1, 0) AS is_view, + IF(STARTSWITH(location, "/dbfs/") + AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root, + IF(UPPER(format) = "DELTA", 1, 0) AS is_delta FROM $inventory.tables) GROUP BY `database` ORDER BY `database` \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql index fa2e62b756..8a9368a416 100644 --- a/src/databricks/labs/ucx/assessment/queries/jobs.sql +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -1,12 +1,9 @@ -- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures --- widget title=Jobs, col=3, row=10, size_x=3, size_y=4 +-- widget title=Jobs, col=3, row=10, size_x=3, size_y=8 SELECT job_id, job_name, creator, - CASE - WHEN success=1 THEN "Compatible" - ELSE "Incompatible" - END AS compatible, + IF(success=1, "Compatible", "Incompatible") AS compatible, failures FROM $inventory.jobs WHERE job_name not like '[UCX]%' diff --git a/src/databricks/labs/ucx/framework/dashboards.py b/src/databricks/labs/ucx/framework/dashboards.py index 44c38eadbc..cfc10ec3d9 100644 --- a/src/databricks/labs/ucx/framework/dashboards.py +++ b/src/databricks/labs/ucx/framework/dashboards.py @@ -227,7 +227,7 @@ def _install_viz(self, query: SimpleQuery): self._state[query.viz_key] = viz.id def _get_viz_options(self, query: SimpleQuery): - viz_types = {"table": self._table_viz_args} + viz_types = {"table": self._table_viz_args, "counter": self._counter_viz_args} if query.viz_type not in viz_types: msg = f"{query.query}: unknown viz type: {query.viz_type}" raise SyntaxError(msg) @@ -270,6 +270,38 @@ def _table_viz_args( }, } + @staticmethod + def _counter_viz_args( + name: str, + value_column: str, + *, + description: str | None = None, + counter_label: str | None = None, + value_row_number: int = 1, + target_row_number: int = 1, + string_decimal: int = 0, + string_decimal_char: str = ".", + string_thousand_separator: str = ",", + tooltip_format: str = "0,0.000", + count_row: bool = False, + ) -> dict: + return { + "type": "COUNTER", + "name": name, + "description": description, + "options": { + "counterLabel": counter_label, + "counterColName": value_column, + "rowNumber": value_row_number, + "targetRowNumber": target_row_number, + "stringDecimal": string_decimal, + "stringDecChar": string_decimal_char, + "stringThouSep": string_thousand_separator, + "tooltipFormat": tooltip_format, + "countRow": count_row, + }, + } + @staticmethod def _parse_magic_comment(f, magic_comment, text): viz_comment = next(_ for _ in text.splitlines() if _.startswith(magic_comment)) diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index 81bb2dc11b..6251a15820 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -1,12 +1,9 @@ import os -from unittest.mock import Mock import pytest from databricks.sdk import WorkspaceClient from databricks.sdk.service.sql import AccessControl, ObjectTypePlural, PermissionLevel -from databricks.labs.ucx.framework.dashboards import DashboardFromFiles -from databricks.labs.ucx.install import Installer from databricks.labs.ucx.mixins.redash import ( DashboardWidgetsAPI, QueryVisualizationsExt, @@ -61,4 +58,3 @@ def test_creating_widgets(ws: WorkspaceClient, make_warehouse, make_schema): y = query_visualizations_api.create_table(query.id, "ABC Viz", [VizColumn(name="databaseName", title="DB")]) print(y) - From 17347f08edb2b3ffad2ef09ef0dfce50f20eb8eb Mon Sep 17 00:00:00 2001 From: Lars George Date: Mon, 25 Sep 2023 14:26:03 +0200 Subject: [PATCH 11/11] Applied feedback, rearranged dashboard, fixed empty widget title --- src/databricks/labs/ucx/assessment/queries/all_tables.sql | 5 ++--- src/databricks/labs/ucx/assessment/queries/clusters.sql | 5 +++-- .../labs/ucx/assessment/queries/count_total_databases.sql | 4 ++++ .../labs/ucx/assessment/queries/count_total_tables.sql | 2 +- .../labs/ucx/assessment/queries/database_summary.sql | 5 ++--- .../labs/ucx/assessment/queries/external_locations.sql | 2 +- src/databricks/labs/ucx/assessment/queries/jobs.sql | 2 +- src/databricks/labs/ucx/assessment/queries/mount_points.sql | 2 +- src/databricks/labs/ucx/framework/dashboards.py | 2 +- 9 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 src/databricks/labs/ucx/assessment/queries/count_total_databases.sql diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 492b90a058..0a8fd95604 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location --- widget title=Table Types, col=0, row=0, size_x=6, size_y=6 +-- widget title=Table Types, col=0, row=3, size_x=6, size_y=6 SELECT `database`, name, object_type AS type, @@ -8,8 +8,7 @@ SELECT `database`, CASE WHEN STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt") THEN "DBFS ROOT" - WHEN STARTSWITH(location, "/dbfs/") - AND STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT" + WHEN STARTSWITH(location, "/dbfs/mnt") THEN "DBFS MOUNT" ELSE "EXTERNAL" END AS storage, IF(format = "delta", "Yes", "No") AS is_delta, diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql index 56baee5bab..98aed74b7c 100644 --- a/src/databricks/labs/ucx/assessment/queries/clusters.sql +++ b/src/databricks/labs/ucx/assessment/queries/clusters.sql @@ -1,8 +1,9 @@ -- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures --- widget title=Clusters, col=0, row=15, size_x=3, size_y=8 +-- widget title=Clusters, col=0, row=25, size_x=6, size_y=8 SELECT cluster_id, cluster_name, creator, IF(success = 1, "Compatible", "Incompatible") AS compatible, failures -FROM $inventory.clusters \ No newline at end of file +FROM $inventory.clusters +WHERE NOT STARTSWITH(cluster_name, "job-") \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/count_total_databases.sql b/src/databricks/labs/ucx/assessment/queries/count_total_databases.sql new file mode 100644 index 0000000000..0eb1d59c61 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/count_total_databases.sql @@ -0,0 +1,4 @@ +-- viz type=counter, name=Total Database Count, counter_label=Total Databases, value_column=count_total_databases +-- widget col=0, row=0, size_x=1, size_y=3 +SELECT COUNT(DISTINCT `database`) AS count_total_databases +FROM hive_metastore.ucx.tables diff --git a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql index 2989f0e486..a346849396 100644 --- a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql @@ -1,4 +1,4 @@ -- viz type=counter, name=Total Table Count, counter_label=Total Tables, value_column=count_total_tables --- widget title=Total Table Count, col=0, row=0, size_x=1, size_y=3 +-- widget col=1, row=0, size_x=1, size_y=3 SELECT count(*) AS count_total_tables FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index df089863d4..18b16cd409 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgrade --- widget title=Database Summary, col=0, row=6, size_x=3, size_y=8 +-- widget title=Database Summary, col=0, row=9, size_x=6, size_y=8 SELECT `database`, SUM(is_table) AS tables, SUM(is_view) AS views, @@ -18,8 +18,7 @@ FROM LOCATION, IF(object_type IN ("MANAGED", "EXTERNAL"), 1, 0) AS is_table, IF(object_type = "VIEW", 1, 0) AS is_view, - IF(STARTSWITH(location, "/dbfs/") - AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root, + IF(STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root, IF(UPPER(format) = "DELTA", 1, 0) AS is_delta FROM $inventory.tables) GROUP BY `database` diff --git a/src/databricks/labs/ucx/assessment/queries/external_locations.sql b/src/databricks/labs/ucx/assessment/queries/external_locations.sql index 4aaf82e09e..7df99099b3 100644 --- a/src/databricks/labs/ucx/assessment/queries/external_locations.sql +++ b/src/databricks/labs/ucx/assessment/queries/external_locations.sql @@ -1,4 +1,4 @@ -- viz type=table, name=External Locations, columns=location --- widget title=External Locations, col=3, row=6, size_x=3, size_y=4 +-- widget title=External Locations, col=0, row=17, size_x=3, size_y=8 SELECT location FROM hive_metastore.ucx.external_locations diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql index 8a9368a416..5842052855 100644 --- a/src/databricks/labs/ucx/assessment/queries/jobs.sql +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures --- widget title=Jobs, col=3, row=10, size_x=3, size_y=8 +-- widget title=Jobs, col=0, row=33, size_x=6, size_y=8 SELECT job_id, job_name, creator, diff --git a/src/databricks/labs/ucx/assessment/queries/mount_points.sql b/src/databricks/labs/ucx/assessment/queries/mount_points.sql index 3ae7e45c55..4ff9fb9c1a 100644 --- a/src/databricks/labs/ucx/assessment/queries/mount_points.sql +++ b/src/databricks/labs/ucx/assessment/queries/mount_points.sql @@ -1,5 +1,5 @@ -- viz type=table, name=Mount Points, columns=name,source --- widget title=Mount Points, col=3, row=10, size_x=3, size_y=4 +-- widget title=Mount Points, col=3, row=17, size_x=3, size_y=8 SELECT name, source FROM hive_metastore.ucx.mounts \ No newline at end of file diff --git a/src/databricks/labs/ucx/framework/dashboards.py b/src/databricks/labs/ucx/framework/dashboards.py index cfc10ec3d9..1418be28c8 100644 --- a/src/databricks/labs/ucx/framework/dashboards.py +++ b/src/databricks/labs/ucx/framework/dashboards.py @@ -130,7 +130,7 @@ def _install_widget(self, query: SimpleQuery): def _get_widget_options(self, query: SimpleQuery): self._pos += 1 widget_options = WidgetOptions( - title=query.widget["title"], + title=query.widget.get("title", ""), description=query.widget.get("description", None), position=WidgetPosition( col=int(query.widget.get("col", 0)),