diff --git a/src/databricks/labs/ucx/hive_metastore/table_migrate.py b/src/databricks/labs/ucx/hive_metastore/table_migrate.py index 3dedc12e78..3edbe5a38a 100644 --- a/src/databricks/labs/ucx/hive_metastore/table_migrate.py +++ b/src/databricks/labs/ucx/hive_metastore/table_migrate.py @@ -293,7 +293,12 @@ def _catalog_type(self): def _catalog_table(self): return self._spark._jvm.org.apache.spark.sql.catalyst.catalog.CatalogTable # pylint: disable=protected-access - def _convert_hms_table_to_external(self, src_table: Table): + def _convert_hms_table_to_external(self, src_table: Table) -> bool: + """Converts a Hive metastore table to external using Spark JVM methods. + + TODO: + This method fails for Databricks runtime 16.0, probably due to the JDK update (https://docs.databricks.com/en/release-notes/runtime/16.0.html#breaking-change-jdk-17-is-now-the-default). + """ logger.info(f"Changing HMS managed table {src_table.name} to External Table type.") inventory_table = self._tables_crawler.full_name try: diff --git a/src/databricks/labs/ucx/installer/policy.py b/src/databricks/labs/ucx/installer/policy.py index 9e2d5efc44..fa029c4858 100644 --- a/src/databricks/labs/ucx/installer/policy.py +++ b/src/databricks/labs/ucx/installer/policy.py @@ -106,7 +106,7 @@ def _get_instance_pool_id(self) -> str | None: return None def _definition(self, conf: dict, instance_profile: str | None, instance_pool_id: str | None) -> str: - latest_lts_dbr = self._ws.clusters.select_spark_version(latest=True) + latest_lts_dbr = self._ws.clusters.select_spark_version(latest=True, long_term_support=True) node_type_id = self._ws.clusters.select_node_type(local_disk=True, min_memory_gb=32, min_cores=4) policy_definition = { "spark_version": self._policy_config(latest_lts_dbr), diff --git a/tests/integration/hive_metastore/test_workflows.py b/tests/integration/hive_metastore/test_workflows.py index 6f065134f3..864a7311be 100644 --- a/tests/integration/hive_metastore/test_workflows.py +++ b/tests/integration/hive_metastore/test_workflows.py @@ -71,6 +71,12 @@ def test_table_migration_job_refreshes_migration_status( def test_table_migration_convert_manged_to_external(installation_ctx, make_table_migration_context) -> None: + """Convert managed tables to external before migrating. + + Note: + This test fails from Databricks runtime 16.0 (https://docs.databricks.com/en/release-notes/runtime/16.0.html), + probably due to the JDK update (https://docs.databricks.com/en/release-notes/runtime/16.0.html#breaking-change-jdk-17-is-now-the-default). + """ tables, dst_schema = make_table_migration_context("managed", installation_ctx) ctx = installation_ctx.replace( config_transform=lambda wc: dataclasses.replace( diff --git a/tests/integration/install/test_installation.py b/tests/integration/install/test_installation.py index 1d275958a5..d33ae53be5 100644 --- a/tests/integration/install/test_installation.py +++ b/tests/integration/install/test_installation.py @@ -108,7 +108,7 @@ def test_job_failure_propagates_correct_error_message_and_logs(ws, sql_backend, @retried(on=[NotFound, InvalidParameterValue], timeout=timedelta(minutes=3)) -def test_job_cluster_policy(ws, installation_ctx): +def test_job_cluster_policy(ws, installation_ctx) -> None: installation_ctx.workspace_installation.run() user_name = ws.current_user.me().user_name cluster_policy = ws.cluster_policies.get(policy_id=installation_ctx.config.policy_id) @@ -116,7 +116,7 @@ def test_job_cluster_policy(ws, installation_ctx): assert cluster_policy.name == f"Unity Catalog Migration ({installation_ctx.inventory_database}) ({user_name})" - spark_version = ws.clusters.select_spark_version(latest=True) + spark_version = ws.clusters.select_spark_version(latest=True, long_term_support=True) assert policy_definition["spark_version"]["value"] == spark_version assert policy_definition["node_type_id"]["value"] == ws.clusters.select_node_type(local_disk=True, min_memory_gb=32) if ws.config.is_azure: