From 1736da9576c3d623d1cf468696a9e3591a5c817f Mon Sep 17 00:00:00 2001 From: Anu Sudarsan Date: Mon, 25 Mar 2024 13:18:07 -0400 Subject: [PATCH] Migrate deltalake/databricks tests to native filesystem --- .../multinode-databricks-http-hms/delta.properties | 4 +++- .../multinode-databricks-http-hms/hive.properties | 4 +++- .../singlenode-delta-lake-databricks/delta.properties | 5 +++-- .../singlenode-delta-lake-databricks/hive.properties | 5 +++-- .../io/trino/tests/product/deltalake/TestDeltaLakeJmx.java | 3 --- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/delta.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/delta.properties index 1a672e49ce05..7b30302c3c9b 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/delta.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/delta.properties @@ -3,6 +3,8 @@ hive.metastore.uri=https://${ENV:DATABRICKS_HOST}:443/api/2.0/unity-hms-proxy/me hive.metastore.http.client.bearer-token=${ENV:DATABRICKS_TOKEN} hive.metastore.http.client.additional-headers=X-Databricks-Catalog-Name:${ENV:DATABRICKS_UNITY_CATALOG_NAME} hive.metastore.http.client.authentication.type=BEARER +fs.hadoop.enabled=false +fs.native-s3.enabled=true # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks -hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL +s3.canned-acl=BUCKET_OWNER_FULL_CONTROL delta.enable-non-concurrent-writes=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/hive.properties index 0a448531d770..7958f9db27e3 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-databricks-http-hms/hive.properties @@ -3,5 +3,7 @@ hive.metastore.uri=https://${ENV:DATABRICKS_HOST}:443/api/2.0/unity-hms-proxy/me hive.metastore.http.client.bearer-token=${ENV:DATABRICKS_TOKEN} hive.metastore.http.client.additional-headers=X-Databricks-Catalog-Name:${ENV:DATABRICKS_UNITY_CATALOG_NAME} hive.metastore.http.client.authentication.type=BEARER +fs.hadoop.enabled=false +fs.native-s3.enabled=true # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks -hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL +s3.canned-acl=BUCKET_OWNER_FULL_CONTROL diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties index 626c55904e0b..fafeb86a3268 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties @@ -1,8 +1,9 @@ connector.name=delta_lake hive.metastore=glue hive.metastore.glue.region=${ENV:AWS_REGION} -fs.hadoop.enabled=true +fs.hadoop.enabled=false +fs.native-s3.enabled=true # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks -hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL +s3.canned-acl=BUCKET_OWNER_FULL_CONTROL delta.enable-non-concurrent-writes=true delta.hive-catalog-name=hive diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties index 8c55145bbc37..2d7e969cb2db 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties @@ -1,9 +1,10 @@ connector.name=hive hive.metastore=glue hive.metastore.glue.region=${ENV:AWS_REGION} -fs.hadoop.enabled=true +fs.hadoop.enabled=false +fs.native-s3.enabled=true # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks -hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL +s3.canned-acl=BUCKET_OWNER_FULL_CONTROL hive.non-managed-table-writes-enabled=true # Required by some product tests hive.hive-views.enabled=true diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java index efbb43416c29..6cc818010b25 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java @@ -34,11 +34,8 @@ public class TestDeltaLakeJmx public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( - row("io.trino.hdfs:name=delta,type=trinofilesystemcachestats"), - row("io.trino.hdfs:name=delta,type=trinohdfsfilesystemstats"), row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), row("io.trino.plugin.hive.metastore.glue:name=delta,type=gluehivemetastore"), - row("io.trino.hdfs.s3:name=delta,type=trinos3filesystem"), row("io.trino.plugin.hive:catalog=delta,name=delta,type=fileformatdatasourcestats"), row("trino.plugin.deltalake.transactionlog:catalog=delta,name=delta,type=transactionlogaccess")); }