From 5120f3fe019f8e6ee2f94d6e954c7bf11ad63f02 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Wed, 3 Aug 2022 20:17:33 +0530 Subject: [PATCH] Allow configuring credential-cache for kerberized hive connector --- .../authentication/AuthenticationModules.java | 12 +-- .../authentication/HdfsKerberosConfig.java | 29 ++++++- .../TestHdfsKerberosConfig.java | 70 +++++++++++++++-- .../authentication/KerberosConfiguration.java | 18 ++++- .../thrift/MetastoreKerberosConfig.java | 29 ++++++- .../ThriftMetastoreAuthenticationModule.java | 6 +- .../thrift/TestMetastoreKerberosConfig.java | 78 +++++++++++++++++-- .../launcher/env/common/HadoopKerberos.java | 5 +- ...sHiveImpersonationWithCredentialCache.java | 50 ++++++++++++ ...iveNoImpersonationWithCredentialCache.java | 50 ++++++++++++ ...sHdfsImpersonationWithCredentialCache.java | 50 ++++++++++++ ...dfsNoImpersonationWithCredentialCache.java | 50 ++++++++++++ .../product/launcher/suite/suites/Suite2.java | 4 + .../product/launcher/suite/suites/Suite5.java | 4 + .../suite/suites/Suite6NonGeneric.java | 8 ++ .../create_kerberos_credential_cache_files.sh | 16 ++++ .../hive.properties | 25 ++++++ .../iceberg.properties | 16 ++++ .../hive.properties | 22 ++++++ .../iceberg.properties | 15 ++++ .../hive.properties | 24 ++++++ .../iceberg.properties | 21 +++++ .../hive.properties | 20 +++++ .../iceberg.properties | 16 ++++ 24 files changed, 609 insertions(+), 29 deletions(-) create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java index 2f8c4e204948..78f3c0c2f90f 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/AuthenticationModules.java @@ -70,10 +70,10 @@ public void configure(Binder binder) HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHdfsTrinoPrincipal(); - String keytabLocation = config.getHdfsTrinoKeytab(); KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() - .withKerberosPrincipal(principal) - .withKeytabLocation(keytabLocation); + .withKerberosPrincipal(principal); + config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation); + config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); return createCachingKerberosHadoopAuthentication(builder.build(), updater); } }; @@ -103,10 +103,10 @@ public void configure(Binder binder) HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHdfsTrinoPrincipal(); - String keytabLocation = config.getHdfsTrinoKeytab(); KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() - .withKerberosPrincipal(principal) - .withKeytabLocation(keytabLocation); + .withKerberosPrincipal(principal); + config.getHdfsTrinoKeytab().ifPresent(builder::withKeytabLocation); + config.getHdfsTrinoCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); return createCachingKerberosHadoopAuthentication(builder.build(), updater); } }; diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java index 652559fd4a17..a068f6e73fda 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/authentication/HdfsKerberosConfig.java @@ -18,12 +18,16 @@ import io.airlift.configuration.LegacyConfig; import io.airlift.configuration.validation.FileExists; +import javax.validation.constraints.AssertTrue; import javax.validation.constraints.NotNull; +import java.util.Optional; + public class HdfsKerberosConfig { private String hdfsTrinoPrincipal; private String hdfsTrinoKeytab; + private String hdfsTrinoCredentialCacheLocation; @NotNull public String getHdfsTrinoPrincipal() @@ -41,10 +45,9 @@ public HdfsKerberosConfig setHdfsTrinoPrincipal(String hdfsTrinoPrincipal) } @NotNull - @FileExists - public String getHdfsTrinoKeytab() + public Optional<@FileExists String> getHdfsTrinoKeytab() { - return hdfsTrinoKeytab; + return Optional.ofNullable(hdfsTrinoKeytab); } @Config("hive.hdfs.trino.keytab") @@ -55,4 +58,24 @@ public HdfsKerberosConfig setHdfsTrinoKeytab(String hdfsTrinoKeytab) this.hdfsTrinoKeytab = hdfsTrinoKeytab; return this; } + + @NotNull + public Optional<@FileExists String> getHdfsTrinoCredentialCacheLocation() + { + return Optional.ofNullable(hdfsTrinoCredentialCacheLocation); + } + + @Config("hive.hdfs.trino.credential-cache.location") + @ConfigDescription("Trino credential-cache location used to access HDFS") + public HdfsKerberosConfig setHdfsTrinoCredentialCacheLocation(String hdfsTrinoCredentialCacheLocation) + { + this.hdfsTrinoCredentialCacheLocation = hdfsTrinoCredentialCacheLocation; + return this; + } + + @AssertTrue(message = "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified") + public boolean isConfigValid() + { + return getHdfsTrinoKeytab().isPresent() ^ getHdfsTrinoCredentialCacheLocation().isPresent(); + } } diff --git a/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java b/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java index efd9aa2e335c..18648d8939e6 100644 --- a/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java +++ b/lib/trino-hdfs/src/test/java/io/trino/hdfs/authentication/TestHdfsKerberosConfig.java @@ -14,16 +14,19 @@ package io.trino.hdfs.authentication; import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.ConfigurationFactory; import org.testng.annotations.Test; -import java.io.IOException; +import javax.validation.constraints.AssertTrue; + import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; +import static org.assertj.core.api.Assertions.assertThat; public class TestHdfsKerberosConfig { @@ -32,12 +35,13 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(HdfsKerberosConfig.class) .setHdfsTrinoPrincipal(null) - .setHdfsTrinoKeytab(null)); + .setHdfsTrinoKeytab(null) + .setHdfsTrinoCredentialCacheLocation(null)); } @Test - public void testExplicitPropertyMappings() - throws IOException + public void testExplicitPropertyMappingsForKeytab() + throws Exception { Path keytab = Files.createTempFile(null, null); @@ -46,10 +50,64 @@ public void testExplicitPropertyMappings() .put("hive.hdfs.trino.keytab", keytab.toString()) .buildOrThrow(); + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class); + HdfsKerberosConfig expected = new HdfsKerberosConfig() .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") .setHdfsTrinoKeytab(keytab.toString()); - assertFullMapping(properties, expected); + assertThat(config.getHdfsTrinoPrincipal()) + .isEqualTo(expected.getHdfsTrinoPrincipal()); + assertThat(config.getHdfsTrinoKeytab()) + .isEqualTo(expected.getHdfsTrinoKeytab()); + } + + @Test + public void testExplicitPropertyMappingsForCredentialCache() + throws Exception + { + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + Map properties = ImmutableMap.builder() + .put("hive.hdfs.trino.principal", "trino@EXAMPLE.COM") + .put("hive.hdfs.trino.credential-cache.location", credentialCacheLocation.toString()) + .buildOrThrow(); + + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + HdfsKerberosConfig config = configurationFactory.build(HdfsKerberosConfig.class); + + HdfsKerberosConfig expected = new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") + .setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString()); + + assertThat(config.getHdfsTrinoPrincipal()) + .isEqualTo(expected.getHdfsTrinoPrincipal()); + assertThat(config.getHdfsTrinoCredentialCacheLocation()) + .isEqualTo(expected.getHdfsTrinoCredentialCacheLocation()); + } + + @Test + public void testValidation() + throws Exception + { + assertFailsValidation( + new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM"), + "configValid", + "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified", + AssertTrue.class); + + Path keytab = Files.createTempFile(null, null); + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + assertFailsValidation( + new HdfsKerberosConfig() + .setHdfsTrinoPrincipal("trino@EXAMPLE.COM") + .setHdfsTrinoKeytab(keytab.toString()) + .setHdfsTrinoCredentialCacheLocation(credentialCacheLocation.toString()), + "configValid", + "Exactly one of `hive.hdfs.trino.keytab` or `hive.hdfs.trino.credential-cache.location` must be specified", + AssertTrue.class); } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java index dadefb39546d..7df9280473f7 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/authentication/KerberosConfiguration.java @@ -76,6 +76,7 @@ public static class Builder { private KerberosPrincipal kerberosPrincipal; private Optional keytabLocation = Optional.empty(); + private Optional credentialCacheLocation = Optional.empty(); public Builder withKerberosPrincipal(String kerberosPrincipal) { @@ -90,6 +91,13 @@ public Builder withKeytabLocation(String keytabLocation) return this; } + public Builder withCredentialCacheLocation(String credentialCacheLocation) + { + verifyFile(credentialCacheLocation); + this.credentialCacheLocation = Optional.of(credentialCacheLocation); + return this; + } + public KerberosConfiguration build() { ImmutableMap.Builder optionsBuilder = ImmutableMap.builder() @@ -97,12 +105,20 @@ public KerberosConfiguration build() .put("isInitiator", "true") .put("principal", kerberosPrincipal.getName()); + checkArgument(keytabLocation.isPresent() ^ credentialCacheLocation.isPresent(), "Either keytab or credential cache must be specified"); + keytabLocation.ifPresent( keytab -> optionsBuilder - .put("storeKey", "true") .put("useKeyTab", "true") + .put("storeKey", "true") .put("keyTab", keytab)); + credentialCacheLocation.ifPresent( + credentialCache -> optionsBuilder + .put("useTicketCache", "true") + .put("renewTGT", "true") + .put("ticketCache", credentialCache)); + return new KerberosConfiguration(kerberosPrincipal, optionsBuilder.buildOrThrow()); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java index 1983938bf288..1aa1ae1eb971 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/MetastoreKerberosConfig.java @@ -17,13 +17,17 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.validation.FileExists; +import javax.validation.constraints.AssertTrue; import javax.validation.constraints.NotNull; +import java.util.Optional; + public class MetastoreKerberosConfig { private String hiveMetastoreServicePrincipal; private String hiveMetastoreClientPrincipal; private String hiveMetastoreClientKeytab; + private String hiveMetastoreCredentialCachePath; @NotNull public String getHiveMetastoreServicePrincipal() @@ -54,10 +58,9 @@ public MetastoreKerberosConfig setHiveMetastoreClientPrincipal(String hiveMetast } @NotNull - @FileExists - public String getHiveMetastoreClientKeytab() + public Optional<@FileExists String> getHiveMetastoreClientKeytab() { - return hiveMetastoreClientKeytab; + return Optional.ofNullable(hiveMetastoreClientKeytab); } @Config("hive.metastore.client.keytab") @@ -67,4 +70,24 @@ public MetastoreKerberosConfig setHiveMetastoreClientKeytab(String hiveMetastore this.hiveMetastoreClientKeytab = hiveMetastoreClientKeytab; return this; } + + @NotNull + public Optional<@FileExists String> getHiveMetastoreClientCredentialCacheLocation() + { + return Optional.ofNullable(hiveMetastoreCredentialCachePath); + } + + @Config("hive.metastore.client.credential-cache.location") + @ConfigDescription("Hive Metastore client credential cache location") + public MetastoreKerberosConfig setHiveMetastoreClientCredentialCacheLocation(String hiveMetastoreCredentialCachePath) + { + this.hiveMetastoreCredentialCachePath = hiveMetastoreCredentialCachePath; + return this; + } + + @AssertTrue(message = "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified") + public boolean isConfigValid() + { + return getHiveMetastoreClientKeytab().isPresent() ^ getHiveMetastoreClientCredentialCacheLocation().isPresent(); + } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java index ae3fe171d77e..fe8114888cbf 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreAuthenticationModule.java @@ -70,10 +70,10 @@ public void configure(Binder binder) public HadoopAuthentication createHadoopAuthentication(MetastoreKerberosConfig config, HdfsConfigurationInitializer updater) { String principal = config.getHiveMetastoreClientPrincipal(); - String keytabLocation = config.getHiveMetastoreClientKeytab(); KerberosConfiguration.Builder builder = new KerberosConfiguration.Builder() - .withKerberosPrincipal(principal) - .withKeytabLocation(keytabLocation); + .withKerberosPrincipal(principal); + config.getHiveMetastoreClientKeytab().ifPresent(builder::withKeytabLocation); + config.getHiveMetastoreClientCredentialCacheLocation().ifPresent(builder::withCredentialCacheLocation); return createCachingKerberosHadoopAuthentication(builder.build(), updater); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java index bd10dea053f1..e4ecb48cfd4c 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestMetastoreKerberosConfig.java @@ -14,16 +14,19 @@ package io.trino.plugin.hive.metastore.thrift; import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.ConfigurationFactory; import org.testng.annotations.Test; -import java.io.IOException; +import javax.validation.constraints.AssertTrue; + import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; +import static org.assertj.core.api.Assertions.assertThat; public class TestMetastoreKerberosConfig { @@ -33,12 +36,13 @@ public void testDefaults() assertRecordedDefaults(recordDefaults(MetastoreKerberosConfig.class) .setHiveMetastoreServicePrincipal(null) .setHiveMetastoreClientPrincipal(null) - .setHiveMetastoreClientKeytab(null)); + .setHiveMetastoreClientKeytab(null) + .setHiveMetastoreClientCredentialCacheLocation(null)); } @Test - public void testExplicitPropertyMappings() - throws IOException + public void testExplicitPropertyMappingsForKeytab() + throws Exception { Path clientKeytabFile = Files.createTempFile(null, null); @@ -48,11 +52,73 @@ public void testExplicitPropertyMappings() .put("hive.metastore.client.keytab", clientKeytabFile.toString()) .buildOrThrow(); + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + MetastoreKerberosConfig config = configurationFactory.build(MetastoreKerberosConfig.class); + MetastoreKerberosConfig expected = new MetastoreKerberosConfig() .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") .setHiveMetastoreClientKeytab(clientKeytabFile.toString()); - assertFullMapping(properties, expected); + assertThat(config.getHiveMetastoreServicePrincipal()) + .isEqualTo(expected.getHiveMetastoreServicePrincipal()); + assertThat(config.getHiveMetastoreClientPrincipal()) + .isEqualTo(expected.getHiveMetastoreClientPrincipal()); + assertThat(config.getHiveMetastoreClientKeytab()) + .isEqualTo(expected.getHiveMetastoreClientKeytab()); + } + + @Test + public void testExplicitPropertyMappingsForCredentialCache() + throws Exception + { + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + Map properties = ImmutableMap.builder() + .put("hive.metastore.service.principal", "hive/_HOST@EXAMPLE.COM") + .put("hive.metastore.client.principal", "metastore@EXAMPLE.COM") + .put("hive.metastore.client.credential-cache.location", credentialCacheLocation.toString()) + .buildOrThrow(); + + ConfigurationFactory configurationFactory = new ConfigurationFactory(properties); + MetastoreKerberosConfig config = configurationFactory.build(MetastoreKerberosConfig.class); + + MetastoreKerberosConfig expected = new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") + .setHiveMetastoreClientCredentialCacheLocation(credentialCacheLocation.toString()); + + assertThat(config.getHiveMetastoreServicePrincipal()) + .isEqualTo(expected.getHiveMetastoreServicePrincipal()); + assertThat(config.getHiveMetastoreClientPrincipal()) + .isEqualTo(expected.getHiveMetastoreClientPrincipal()); + assertThat(config.getHiveMetastoreClientCredentialCacheLocation()) + .isEqualTo(expected.getHiveMetastoreClientCredentialCacheLocation()); + } + + @Test + public void testValidation() + throws Exception + { + assertFailsValidation( + new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM"), + "configValid", + "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified", + AssertTrue.class); + + Path clientKeytabFile = Files.createTempFile(null, null); + Path credentialCacheLocation = Files.createTempFile("credentialCache", null); + + assertFailsValidation( + new MetastoreKerberosConfig() + .setHiveMetastoreServicePrincipal("hive/_HOST@EXAMPLE.COM") + .setHiveMetastoreClientPrincipal("metastore@EXAMPLE.COM") + .setHiveMetastoreClientKeytab(clientKeytabFile.toString()) + .setHiveMetastoreClientCredentialCacheLocation(credentialCacheLocation.toString()), + "configValid", + "Exactly one of `hive.metastore.client.keytab` or `hive.metastore.client.credential-cache.location` must be specified", + AssertTrue.class); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java index e37410c55fbd..7320ff69b960 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HadoopKerberos.java @@ -69,7 +69,10 @@ public void extendEnvironment(Environment.Builder builder) portBinder.exposePort(container, 7778); container .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) - .withCopyFileToContainer(forHostPath(configDir.getPath("config.properties")), CONTAINER_PRESTO_CONFIG_PROPERTIES); + .withCopyFileToContainer(forHostPath(configDir.getPath("config.properties")), CONTAINER_PRESTO_CONFIG_PROPERTIES) + .withCopyFileToContainer( + forHostPath(configDir.getPath("create_kerberos_credential_cache_files.sh")), + "/docker/presto-init.d/create_kerberos_credentials.sh"); }); builder.configureContainer(TESTS, container -> { container.setDockerImageName(dockerImageName); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..0dc3be3b0a42 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberos; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosHiveImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosHiveImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberos hadoopKerberos) + { + super(ImmutableList.of(standard, hadoopKerberos)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..dd4de06080d6 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberos; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberos hadoopKerberos) + { + super(ImmutableList.of(standard, hadoopKerberos)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..caf896851375 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberosKmsWithImpersonation; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberosKmsWithImpersonation hadoopKerberosKmsWithImpersonation) + { + super(ImmutableList.of(standard, hadoopKerberosKmsWithImpersonation)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java new file mode 100644 index 000000000000..3b5347b8342f --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.HadoopKerberosKms; +import io.trino.tests.product.launcher.env.common.Standard; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache(DockerFiles dockerFiles, Standard standard, HadoopKerberosKms hadoopKerberosKms) + { + super(ImmutableList.of(standard, hadoopKerberosKms)); + configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache"); + } + + @Override + @SuppressWarnings("resource") + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + builder.addConnector("iceberg", forHostPath(configDir.getPath("iceberg.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java index b19764c72a12..1d410d7ec104 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite2.java @@ -19,6 +19,7 @@ import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHdfsImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsNoImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -39,6 +40,9 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosHdfsNoImpersonation.class) .withGroups("configured_features", "storage_formats", "hdfs_no_impersonation") .build(), + testOnEnvironment(EnvSinglenodeKerberosHiveNoImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats", "hdfs_no_impersonation") + .build(), testOnEnvironment(EnvSinglenodeHdfsImpersonation.class) .withGroups("configured_features", "storage_formats", "cli", "hdfs_impersonation") .build(), diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java index 90c89177d943..5ebb6ba4177a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite5.java @@ -18,6 +18,7 @@ import io.trino.tests.product.launcher.env.environment.EnvMultinodeHiveCaching; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHiveImpersonation; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHiveImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -38,6 +39,9 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosHiveImpersonation.class) .withGroups("configured_features", "storage_formats", "hdfs_impersonation", "authorization") .build(), + testOnEnvironment(EnvSinglenodeKerberosHiveImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats", "hdfs_impersonation", "authorization") + .build(), testOnEnvironment(EnvMultinodeHiveCaching.class) .withGroups("configured_features", "hive_caching", "storage_formats") .withExcludedGroups("iceberg") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java index 013d98051323..164bcd0de949 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java @@ -22,7 +22,9 @@ import io.trino.tests.product.launcher.env.environment.EnvMultinodePhoenix5; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeCassandra; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsNoImpersonation; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -43,9 +45,15 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeKerberosKmsHdfsNoImpersonation.class) .withGroups("configured_features", "storage_formats") .build(), + testOnEnvironment(EnvSinglenodeKerberosKmsHdfsNoImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats") + .build(), testOnEnvironment(EnvSinglenodeKerberosKmsHdfsImpersonation.class) .withGroups("configured_features", "storage_formats") .build(), + testOnEnvironment(EnvSinglenodeKerberosKmsHdfsImpersonationWithCredentialCache.class) + .withGroups("configured_features", "storage_formats") + .build(), testOnEnvironment(EnvSinglenodeCassandra.class) .withGroups("configured_features", "cassandra") .build(), diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh new file mode 100644 index 000000000000..e74d7fd60260 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop-kerberos/create_kerberos_credential_cache_files.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +kinit -f -c /etc/trino/conf/presto-server-krbcc \ + -kt /etc/trino/conf/presto-server.keytab presto-server/$(hostname -f)@LABS.TERADATA.COM + +kinit -f -c /etc/trino/conf/hive-presto-master-krbcc \ + -kt /etc/trino/conf/hive-presto-master.keytab hive/$(hostname -f)@LABS.TERADATA.COM + + +kinit -f -c /etc/trino/conf/hdfs-krbcc \ + -kt /etc/hadoop/conf/hdfs.keytab hdfs/hadoop-master@LABS.TERADATA.COM + +kinit -f -c /etc/trino/conf/hive-krbcc \ + -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..6b136719230e --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,25 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.metastore-cache-ttl=0s + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-presto-master-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +# Restrictive new directory permissions make sense when HDFS and metastore impersonation is enabled. +# We cannot enable them here, because 'authorization' tests require that one user is able to write to other user's table. +#hive.fs.new-directory-permissions=0700 +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 + +#required for testGrantRevoke() product test +hive.security=sql-standard +#required for testAccessControlSetHiveViewAuthorization() product test +hive.hive-views.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..c376c7179c0c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-presto-master-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..38fbd90a4c30 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,22 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-comment-table=true +hive.metastore-cache-ttl=0s +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/hdfs-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..8182b53463f6 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/hdfs-krbcc + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..01add4b28b47 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,24 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +# When using HDFS impersonation we talk to HDFS as session user configured in tempto (which is 'hive') +# However, when DROP TABLE is issued, Metastore needs to be able to delete the table files on HDFS, +# otherwise they be silently left behind, potentially causing some further tests to fail. For this reason, +# `hive.metastore.client.principal` needs to match session user configured in tempto. +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..ea0a276ecfb2 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,21 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +# When using HDFS impersonation we talk to HDFS as session user configured in tempto (which is 'hive') +# However, TODO we don't have Metastore impersonation yet; when DROP TABLE is issued, +# Metastore needs to be able to delete the table files on HDFS, otherwise they be silently left behind, +# potentially causing some further tests to fail. For this reason, `hive.metastore.client.principal` needs +# to match session user configured in tempto. +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/hive-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties new file mode 100644 index 000000000000..15a8a7c69458 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties @@ -0,0 +1,20 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties new file mode 100644 index 000000000000..76308bbc3125 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.metastore.client.credential-cache.location=/etc/trino/conf/presto-server-krbcc + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.trino.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.trino.credential-cache.location=/etc/trino/conf/presto-server-krbcc +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop-kms/conf/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET