diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 0785b435afc0..e2bec0d34ec9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -1041,7 +1041,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe external = true; targetPath = Optional.of(getValidatedExternalLocation(externalLocation)); - checkExternalPath(session, targetPath.get()); + checkExternalPathAndCreateIfNotExists(session, targetPath.get()); } else { external = false; @@ -1319,11 +1319,16 @@ private static Location getValidatedExternalLocation(String location) return validated; } - private void checkExternalPath(ConnectorSession session, Location location) + private void checkExternalPathAndCreateIfNotExists(ConnectorSession session, Location location) { try { if (!fileSystemFactory.create(session).directoryExists(location).orElse(true)) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "External location must be a directory: " + location); + if (writesToNonManagedTablesEnabled) { + createDirectory(session, location); + } + else { + throw new TrinoException(INVALID_TABLE_PROPERTY, "External location must be a directory: " + location); + } } } catch (IOException | IllegalArgumentException e) { @@ -1331,6 +1336,16 @@ private void checkExternalPath(ConnectorSession session, Location location) } } + private void createDirectory(ConnectorSession session, Location location) + { + try { + fileSystemFactory.create(session).createDirectory(location); + } + catch (IOException e) { + throw new TrinoException(INVALID_TABLE_PROPERTY, e.getMessage()); + } + } + private void checkPartitionTypesSupported(List partitionColumns) { for (HiveColumnHandle partitionColumn : partitionColumns) { diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java index 3926e54024cf..63e70d2fcf42 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java @@ -53,6 +53,7 @@ import io.trino.plugin.hive.security.SqlStandardAccessControlMetadata; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorInsertTableHandle; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorOutputTableHandle; import io.trino.spi.connector.ConnectorPageSink; @@ -90,6 +91,7 @@ import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; @@ -105,6 +107,7 @@ import static io.trino.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata; import static io.trino.plugin.hive.AbstractTestHive.getAllSplits; import static io.trino.plugin.hive.AbstractTestHive.getSplits; +import static io.trino.plugin.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS; import static io.trino.plugin.hive.HiveTestUtils.PAGE_SORTER; import static io.trino.plugin.hive.HiveTestUtils.SESSION; @@ -122,6 +125,7 @@ import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.testing.MaterializedResult.materializeSourceDataStream; import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; +import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.TestingPageSinkId.TESTING_PAGE_SINK_ID; import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static java.nio.charset.StandardCharsets.UTF_8; @@ -143,6 +147,7 @@ public abstract class AbstractTestHiveFileSystem protected SchemaTableName tableWithHeader; protected SchemaTableName tableWithHeaderAndFooter; protected SchemaTableName temporaryCreateTable; + protected SchemaTableName temporaryCreateTableWithExternalLocation; protected HdfsEnvironment hdfsEnvironment; protected LocationService locationService; @@ -190,8 +195,11 @@ protected void setup(String host, int port, String databaseName, boolean s3Selec String random = randomUUID().toString().toLowerCase(ENGLISH).replace("-", ""); temporaryCreateTable = new SchemaTableName(database, "tmp_trino_test_create_" + random); + temporaryCreateTableWithExternalLocation = new SchemaTableName(database, "tmp_trino_test_create_external" + random); - config = new HiveConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled); + config = new HiveConfig() + .setWritesToNonManagedTablesEnabled(true) + .setS3SelectPushdownEnabled(s3SelectPushdownEnabled); HivePartitionManager hivePartitionManager = new HivePartitionManager(config); @@ -670,6 +678,24 @@ public void testTableCreation() } } + @Test + public void testTableCreationExternalLocation() + throws Exception + { + for (HiveStorageFormat storageFormat : HiveStorageFormat.values()) { + if (storageFormat == HiveStorageFormat.CSV) { + // CSV supports only unbounded VARCHAR type + continue; + } + if (storageFormat == HiveStorageFormat.REGEX) { + // REGEX format is read-only + continue; + } + createExternalTableOnNonExistingPath(temporaryCreateTableWithExternalLocation, storageFormat); + dropTable(temporaryCreateTableWithExternalLocation); + } + } + private void createTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception { @@ -734,6 +760,84 @@ private void createTable(SchemaTableName tableName, HiveStorageFormat storageFor } } + private void createExternalTableOnNonExistingPath(SchemaTableName tableName, HiveStorageFormat storageFormat) + throws Exception + { + List columns = ImmutableList.of(new ColumnMetadata("id", BIGINT)); + String externalLocation = getBasePath() + "/external_" + randomNameSuffix(); + + MaterializedResult data = MaterializedResult.resultBuilder(newSession(), BIGINT) + .row(1L) + .row(3L) + .row(2L) + .build(); + + try (Transaction transaction = newTransaction()) { + ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(); + + Map tableProperties = ImmutableMap.builder() + .putAll(createTableProperties(storageFormat)) + .put(EXTERNAL_LOCATION_PROPERTY, externalLocation) + .buildOrThrow(); + + // begin creating the table + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, tableProperties); + metadata.createTable(session, tableMetadata, true); + + transaction.commit(); + + // Hack to work around the metastore not being configured for S3 or other FS. + // The metastore tries to validate the location when creating the + // table, which fails without explicit configuration for file system. + // We work around that by using a dummy location when creating the + // table and update it here to the correct location. + Location location = locationService.getTableWriteInfo(new LocationHandle(externalLocation, externalLocation, LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY), false).targetPath(); + metastoreClient.updateTableLocation(database, tableName.getTableName(), location.toString()); + } + + try (Transaction transaction = newTransaction()) { + ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(); + + ConnectorTableHandle connectorTableHandle = getTableHandle(metadata, tableName); + ConnectorInsertTableHandle outputHandle = metadata.beginInsert(session, connectorTableHandle, ImmutableList.of(), NO_RETRIES); + + ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle, TESTING_PAGE_SINK_ID); + sink.appendPage(data.toPage()); + Collection fragments = getFutureValue(sink.finish()); + + metadata.finishInsert(session, outputHandle, fragments, ImmutableList.of()); + transaction.commit(); + } + + try (Transaction transaction = newTransaction()) { + ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(); + + // load the new table + ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); + List columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values()); + + // verify the metadata + ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName)); + assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns); + assertEquals(tableMetadata.getProperties().get("external_location"), externalLocation); + + // verify the data + metadata.beginQuery(session); + ConnectorSplitSource splitSource = getSplits(splitManager, transaction, session, tableHandle); + ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); + + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles, DynamicFilter.EMPTY)) { + MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); + assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows()); + } + + metadata.cleanupQuery(session); + } + } + private void dropTable(SchemaTableName table) { try (Transaction transaction = newTransaction()) { diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCreateExternalTable.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCreateExternalTable.java index 6ca1d28257b5..d8b590ffc8ad 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCreateExternalTable.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveCreateExternalTable.java @@ -21,17 +21,20 @@ import org.intellij.lang.annotations.Language; import org.testng.annotations.Test; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; +import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tpch.TpchTable.CUSTOMER; import static io.trino.tpch.TpchTable.ORDERS; import static java.lang.String.format; import static java.nio.file.Files.createTempDirectory; import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; public class TestHiveCreateExternalTable extends AbstractTestQueryFramework @@ -87,4 +90,58 @@ public void testCreateExternalTableAsWithExistingDirectory() assertQueryFails(createTableSql, "Target directory for table '.*' already exists:.*"); } + + @Test + public void testCreateExternalTableOnNonExistingPath() + throws Exception + { + java.nio.file.Path tempDir = createTempDirectory(null); + // delete dir, trino should recreate it + deleteRecursively(tempDir, ALLOW_INSECURE); + String tableName = "test_create_external_non_exists_" + randomNameSuffix(); + + @Language("SQL") String createTableSql = format("" + + "CREATE TABLE %s.%s.%s (\n" + + " col1 varchar,\n" + + " col2 varchar\n" + + ")\n" + + "WITH (\n" + + " external_location = '%s',\n" + + " format = 'TEXTFILE'\n" + + ")", + getSession().getCatalog().get(), + getSession().getSchema().get(), + tableName, + tempDir.toUri().toASCIIString()); + + assertUpdate(createTableSql); + String actual = (String) computeScalar("SHOW CREATE TABLE " + tableName); + assertEquals(actual, createTableSql); + assertUpdate("DROP TABLE " + tableName); + deleteRecursively(tempDir, ALLOW_INSECURE); + } + + @Test + public void testCreateExternalTableOnExistingPathToFile() + throws Exception + { + File tempFile = File.createTempFile("temp", ".tmp"); + tempFile.deleteOnExit(); + String tableName = "test_create_external_on_file_" + randomNameSuffix(); + + @Language("SQL") String createTableSql = format(""" + CREATE TABLE %s.%s.%s ( + col1 varchar, + col2 varchar + )WITH ( + external_location = '%s', + format = 'TEXTFILE') + """, + getSession().getCatalog().get(), + getSession().getSchema().get(), + tableName, + tempFile.toPath().toUri().toASCIIString()); + + assertQueryFails(createTableSql, ".*Destination exists and is not a directory.*"); + } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java index d66491bb0821..bc740b467222 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveS3AndGlueMetastoreTest.java @@ -143,6 +143,32 @@ public void testBasicOperationsWithProvidedTableLocation(boolean partitioned, Lo } } + @Test(dataProvider = "locationPatternsDataProvider") + public void testBasicOperationsWithProvidedTableLocationNonCTAS(boolean partitioned, LocationPattern locationPattern) + { + // this test needed, because execution path for CTAS and simple create is different + String tableName = "test_basic_operations_" + randomNameSuffix(); + String location = locationPattern.locationForTable(bucketName, schemaName, tableName); + String partitionQueryPart = (partitioned ? ",partitioned_by = ARRAY['col_int']" : ""); + + String create = "CREATE TABLE " + tableName + "(col_str varchar, col_int integer) WITH (external_location = '" + location + "' " + partitionQueryPart + ")"; + if (locationPattern == DOUBLE_SLASH || locationPattern == TRIPLE_SLASH || locationPattern == TWO_TRAILING_SLASHES) { + assertQueryFails(create, "\\QUnsupported location that cannot be internally represented: " + location); + return; + } + assertUpdate(create); + try (UncheckedCloseable ignored = onClose("DROP TABLE " + tableName)) { + String actualTableLocation = getTableLocation(tableName); + assertThat(actualTableLocation).isEqualTo(location); + + assertUpdate("INSERT INTO " + tableName + " VALUES ('str1', 1), ('str2', 2), ('str3', 3), ('str4', 4)", 4); + assertQuery("SELECT * FROM " + tableName, "VALUES ('str1', 1), ('str2', 2), ('str3', 3), ('str4', 4)"); + + assertThat(getTableFiles(actualTableLocation)).isNotEmpty(); + validateDataFiles(partitioned ? "col_int" : "", tableName, actualTableLocation); + } + } + @Override // Row-level modifications are not supported for Hive tables @Test(dataProvider = "locationPatternsDataProvider") public void testBasicOperationsWithProvidedSchemaLocation(boolean partitioned, LocationPattern locationPattern) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java index 88e8235f3f93..dbf588f70a66 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java @@ -36,6 +36,7 @@ public List getTestRuns(EnvironmentConfig config) return ImmutableList.of( testOnEnvironment(EnvMultinode.class) .withGroups("configured_features", "hdfs_no_impersonation") + .withExcludedTests("io.trino.tests.product.TestImpersonation.testExternalLocationTableCreationSuccess") .build(), testOnEnvironment(EnvSinglenodeKerberosHdfsNoImpersonation.class) .withGroups("configured_features", "storage_formats", "hdfs_no_impersonation", "hive_kerberos") diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties index 3987b3b09440..f3794109bcb5 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties @@ -14,3 +14,4 @@ hive.hdfs.impersonation.enabled=true hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties index a6c83d3f1a87..dd977ce5226a 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties @@ -16,3 +16,4 @@ hive.fs.new-directory-permissions=0700 hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties index a6d7acee72ce..100f9af6b5aa 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties @@ -29,3 +29,4 @@ hive.hdfs.trino.keytab=/etc/hadoop/conf/hdfs-other.keytab hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties index 42eda368ad90..f3c23d43660b 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties @@ -20,3 +20,4 @@ hive.security=sql-standard hive.hive-views.enabled=true hive.hdfs.wire-encryption.enabled=true +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties index df60795c248b..e05fe7a37af1 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties @@ -20,3 +20,4 @@ hive.max-partitions-for-eager-load=100 hive.security=sql-standard #required for testAccessControlSetHiveViewAuthorization() product test hive.hive-views.enabled=true +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties index c28244fea67f..9dc23a5de5ac 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties @@ -21,3 +21,4 @@ hive.hdfs.trino.keytab=/etc/hadoop/conf/hdfs.keytab hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties index 0bd68817f824..e90065ef200f 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties @@ -24,3 +24,4 @@ hive.max-partitions-for-eager-load=100 hive.security=sql-standard #required for testAccessControlSetHiveViewAuthorization() product test hive.hive-views.enabled=true +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties index af6ca5b018d0..b367e08a0885 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties @@ -24,3 +24,4 @@ hive.max-partitions-for-eager-load=100 hive.security=sql-standard #required for testAccessControlSetHiveViewAuthorization() product test hive.hive-views.enabled=true +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties index a6c0271428fc..97b3a12bbc94 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties @@ -21,3 +21,4 @@ hive.hdfs.trino.credential-cache.location=/etc/trino/conf/hdfs-krbcc hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.non-managed-table-writes-enabled=true diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestImpersonation.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestImpersonation.java index ce15a13fd249..ad5d7b8f26f4 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestImpersonation.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestImpersonation.java @@ -26,6 +26,8 @@ import java.net.URI; import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; +import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.HDFS_IMPERSONATION; import static io.trino.tests.product.TestGroups.HDFS_NO_IMPERSONATION; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; @@ -33,12 +35,14 @@ import static io.trino.tests.product.utils.HadoopTestUtils.RETRYABLE_FAILURES_MATCH; import static io.trino.tests.product.utils.QueryExecutors.connectToTrino; import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; public class TestImpersonation extends ProductTest { private QueryExecutor aliceExecutor; + private QueryExecutor bobExecutor; @Inject private HdfsClient hdfsClient; @@ -47,6 +51,10 @@ public class TestImpersonation @Named("databases.alice@presto.jdbc_user") private String aliceJdbcUser; + @Inject + @Named("databases.bob@presto.jdbc_user") + private String bobJdbcUser; + // The value for configuredHdfsUser is profile dependent // For non-Kerberos environments this variable will be equal to -DHADOOP_USER_NAME as set in jvm.config // For Kerberized environments this variable will be equal to the hive.hdfs.trino.principal property as set in hive.properties @@ -54,10 +62,15 @@ public class TestImpersonation @Named("databases.presto.configured_hdfs_user") private String configuredHdfsUser; + @Inject + @Named("databases.hive.warehouse_directory_path") + private String warehouseLocation; + @BeforeMethodWithContext public void setup() { aliceExecutor = connectToTrino("alice@presto"); + bobExecutor = connectToTrino("bob@presto"); } @AfterMethodWithContext @@ -65,6 +78,49 @@ public void cleanup() { // should not be closed, this would close a shared, global QueryExecutor aliceExecutor = null; + bobExecutor = null; + } + + @Test(groups = {HDFS_IMPERSONATION, PROFILE_SPECIFIC_TESTS}) + public void testExternalLocationTableCreationFailure() + { + String commonExternalLocationPath = warehouseLocation + "/nested_" + randomNameSuffix(); + + String tableNameBob = "bob_external_table" + randomNameSuffix(); + String tableLocationBob = commonExternalLocationPath + "/bob_table"; + bobExecutor.executeQuery(format("CREATE TABLE %s (a bigint) WITH (external_location = '%s')", tableNameBob, tableLocationBob)); + String owner = hdfsClient.getOwner(commonExternalLocationPath); + assertEquals(owner, bobJdbcUser); + + String tableNameAlice = "alice_external_table" + randomNameSuffix(); + String tableLocationAlice = commonExternalLocationPath + "/alice_table"; + assertQueryFailure(() -> aliceExecutor.executeQuery(format("CREATE TABLE %s (a bigint) WITH (external_location = '%s')", tableNameAlice, tableLocationAlice))) + .hasStackTraceContaining("Permission denied"); + + bobExecutor.executeQuery(format("DROP TABLE IF EXISTS %s", tableNameBob)); + aliceExecutor.executeQuery(format("DROP TABLE IF EXISTS %s", tableNameAlice)); + } + + @Test(groups = {HDFS_NO_IMPERSONATION, PROFILE_SPECIFIC_TESTS}) + public void testExternalLocationTableCreationSuccess() + { + String commonExternalLocationPath = warehouseLocation + "/nested_" + randomNameSuffix(); + + String tableNameBob = "bob_external_table" + randomNameSuffix(); + String tableLocationBob = commonExternalLocationPath + "/bob_table"; + bobExecutor.executeQuery(format("CREATE TABLE %s (a bigint) WITH (external_location = '%s')", tableNameBob, tableLocationBob)); + String owner = hdfsClient.getOwner(tableLocationBob); + assertEquals(owner, configuredHdfsUser); + + String tableNameAlice = "alice_external_table" + randomNameSuffix(); + String tableLocationAlice = commonExternalLocationPath + "/alice_table"; + aliceExecutor.executeQuery(format("CREATE TABLE %s (a bigint) WITH (external_location = '%s')", tableNameAlice, tableLocationAlice)); + assertThat(aliceExecutor.executeQuery(format("SELECT * FROM %s", tableNameAlice))).hasRowsCount(0); + owner = hdfsClient.getOwner(tableLocationAlice); + assertEquals(owner, configuredHdfsUser); + + bobExecutor.executeQuery(format("DROP TABLE IF EXISTS %s", tableNameBob)); + aliceExecutor.executeQuery(format("DROP TABLE IF EXISTS %s", tableNameAlice)); } @Test(groups = {HDFS_NO_IMPERSONATION, PROFILE_SPECIFIC_TESTS})