From 10e2943996675eaf7c729b63648f9647d76a4c74 Mon Sep 17 00:00:00 2001 From: FANNG Date: Fri, 18 Oct 2024 18:38:22 +0800 Subject: [PATCH] [#4993] feat(iceberg): integrate credential framework to iceberg REST server (#5134) ### What changes were proposed in this pull request? integrate credential framework to iceberg REST server ### Why are the changes needed? Fix: #4993 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? set up a local environment to request credential with the configuration `spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation=vended-credentials` --- .../lakehouse/iceberg/IcebergConstants.java | 5 +- .../credential/CredentialConstants.java | 26 ++ .../iceberg/ops/TestIcebergTableUpdate.java | 4 +- .../credential/CredentialPropertyUtils.java | 38 +++ .../credential/CredentialProviderManager.java | 73 +++++ .../gravitino/credential/CredentialUtils.java | 32 ++ iceberg/iceberg-common/build.gradle.kts | 1 + .../iceberg/common/IcebergConfig.java | 16 +- ...java => IcebergCatalogConfigProvider.java} | 14 +- .../common/ops/IcebergCatalogWrapper.java | 9 +- ... DynamicIcebergCatalogConfigProvider.java} | 25 +- ...> StaticIcebergCatalogConfigProvider.java} | 19 +- .../service/IcebergCatalogWrapperManager.java | 88 +++-- .../service/rest/IcebergTableOperations.java | 99 +++++- ...DynamicIcebergCatalogWrapperProvider.java} | 20 +- ...tStaticIcebergCatalogWrapperProvider.java} | 23 +- .../extension/DummyCredentialProvider.java | 66 ++++ ...dIcebergCatalogWrapperProviderForTest.java | 30 -- .../rest/IcebergCatalogWrapperForTest.java | 6 + .../IcebergCatalogWrapperManagerForTest.java | 37 +++ .../service/rest/IcebergRestTestUtil.java | 16 +- .../rest/TestIcebergTableOperations.java | 302 ++++++++++-------- ...he.gravitino.credential.CredentialProvider | 19 ++ 23 files changed, 711 insertions(+), 257 deletions(-) create mode 100644 catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java create mode 100644 common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java rename iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/{IcebergCatalogWrapperProvider.java => IcebergCatalogConfigProvider.java} (71%) rename iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/{GravitinoBasedIcebergCatalogWrapperProvider.java => DynamicIcebergCatalogConfigProvider.java} (81%) rename iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/{ConfigBasedIcebergCatalogWrapperProvider.java => StaticIcebergCatalogConfigProvider.java} (78%) rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/{TestGravitinoBasedIcebergCatalogWrapperProvider.java => TestDynamicIcebergCatalogWrapperProvider.java} (87%) rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/{TestConfigBasedIcebergCatalogWrapperProvider.java => TestStaticIcebergCatalogWrapperProvider.java} (86%) create mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java delete mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java create mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java create mode 100644 iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 21462b9ca91..004bde0bd7e 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -68,7 +68,10 @@ public class IcebergConstants { public static final String ICEBERG_REST_CATALOG_CACHE_EVICTION_INTERVAL = "catalog-cache-eviction-interval-ms"; - public static final String ICEBERG_REST_CATALOG_PROVIDER = "catalog-provider"; + public static final String ICEBERG_REST_CATALOG_CONFIG_PROVIDER = "catalog-config-provider"; + public static final String STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME = "static-config-provider"; + public static final String DYNAMIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME = + "dynamic-config-provider"; public static final String GRAVITINO_URI = "gravitino-uri"; diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java new file mode 100644 index 00000000000..596268395e3 --- /dev/null +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +public class CredentialConstants { + public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + + private CredentialConstants() {} +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java index c4bac4df124..37124dc5f33 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java @@ -18,9 +18,11 @@ */ package org.apache.gravitino.catalog.lakehouse.iceberg.ops; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper.IcebergTableChange; import org.apache.gravitino.rel.TableChange; @@ -79,7 +81,7 @@ public class TestIcebergTableUpdate { @BeforeEach public void init() { - icebergCatalogWrapper = new IcebergCatalogWrapper(); + icebergCatalogWrapper = new IcebergCatalogWrapper(new IcebergConfig(Collections.emptyMap())); icebergCatalogWrapperHelper = new IcebergCatalogWrapperHelper(icebergCatalogWrapper.getCatalog()); createNamespace(TEST_NAMESPACE_NAME); diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java new file mode 100644 index 00000000000..255e54fbf3d --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Map; + +/** + * Helper class to generate specific credential properties for different table format and engine. + */ +public class CredentialPropertyUtils { + /** + * Transforms a specific credential into a map of Iceberg properties. + * + * @param credential the credential to be transformed into Iceberg properties + * @return a map of Iceberg properties derived from the credential + */ + public static Map toIcebergProperties(Credential credential) { + // todo: transform specific credential to iceberg properties + return credential.toProperties(); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java new file mode 100644 index 00000000000..b583bedcfdf --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CredentialProviderManager { + + private static final Logger LOG = LoggerFactory.getLogger(CredentialProviderManager.class); + private Map credentialProviders; + + public CredentialProviderManager() { + this.credentialProviders = new ConcurrentHashMap<>(); + } + + public void registerCredentialProvider( + String catalogName, CredentialProvider credentialProvider) { + CredentialProvider current = credentialProviders.putIfAbsent(catalogName, credentialProvider); + Preconditions.checkState( + !credentialProvider.equals(current), + String.format( + "Should not register multiple times to CredentialProviderManager, catalog: %s, " + + "credential provider: %s", + catalogName, credentialProvider.credentialType())); + LOG.info( + "Register catalog:%s credential provider:%s to CredentialProviderManager", + catalogName, credentialProvider.credentialType()); + } + + public void unregisterCredentialProvider(String catalogName) { + CredentialProvider credentialProvider = credentialProviders.remove(catalogName); + // Not all catalog has credential provider + if (credentialProvider != null) { + LOG.info( + "Unregister catalog:{} credential provider:{} to CredentialProviderManager", + catalogName, + credentialProvider.credentialType()); + try { + credentialProvider.close(); + } catch (IOException e) { + LOG.warn("Close credential provider failed", e); + } + } + } + + @Nullable + public CredentialProvider getCredentialProvider(String catalogName) { + return credentialProviders.get(catalogName); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java new file mode 100644 index 00000000000..ad81953ac61 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.collect.ImmutableSet; +import org.apache.gravitino.utils.PrincipalUtils; + +public class CredentialUtils { + public static Credential vendCredential(CredentialProvider credentialProvider, String path) { + PathBasedCredentialContext pathBasedCredentialContext = + new PathBasedCredentialContext( + PrincipalUtils.getCurrentUserName(), ImmutableSet.of(path), ImmutableSet.of()); + return credentialProvider.getCredential(pathBasedCredentialContext); + } +} diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index 23b3d30db28..abc9a05a550 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -25,6 +25,7 @@ plugins { } dependencies { + implementation(project(":api")) implementation(project(":catalogs:catalog-common")) implementation(project(":core")) { exclude("*") diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index fd7b52050c3..638b4172ce4 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -32,6 +32,7 @@ import org.apache.gravitino.config.ConfigBuilder; import org.apache.gravitino.config.ConfigConstants; import org.apache.gravitino.config.ConfigEntry; +import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.storage.OSSProperties; import org.apache.gravitino.storage.S3Properties; @@ -201,13 +202,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .longConf() .createWithDefault(3600000L); - public static final ConfigEntry ICEBERG_REST_CATALOG_PROVIDER = - new ConfigBuilder(IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER) + public static final ConfigEntry ICEBERG_REST_CATALOG_CONFIG_PROVIDER = + new ConfigBuilder(IcebergConstants.ICEBERG_REST_CATALOG_CONFIG_PROVIDER) .doc( - "Catalog provider class name, you can develop a class that implements `IcebergCatalogWrapperProvider` and add the corresponding jar file to the Iceberg REST service classpath directory.") + "Catalog provider class name, you can develop a class that implements `IcebergCatalogConfigProvider` and add the corresponding jar file to the Iceberg REST service classpath directory.") .version(ConfigConstants.VERSION_0_7_0) .stringConf() - .createWithDefault("config-based-provider"); + .createWithDefault(IcebergConstants.STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME); public static final ConfigEntry GRAVITINO_URI = new ConfigBuilder(IcebergConstants.GRAVITINO_URI) @@ -233,6 +234,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .toSequence() .createWithDefault(Collections.emptyList()); + public static final ConfigEntry CREDENTIAL_PROVIDER_TYPE = + new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDER_TYPE) + .doc("The credential provider type for Iceberg") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + public String getJdbcDriver() { return get(JDBC_DRIVER); } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java similarity index 71% rename from iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java rename to iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java index 758aa46aa08..fc0d488a11d 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java @@ -19,12 +19,14 @@ package org.apache.gravitino.iceberg.common.ops; import java.util.Map; +import java.util.Optional; +import org.apache.gravitino.iceberg.common.IcebergConfig; /** - * IcebergCatalogWrapperProvider is an interface defining how Iceberg REST catalog server gets - * Iceberg catalogs. + * {@code IcebergCatalogConfigProvider} is an interface defining how Iceberg REST catalog server + * gets Iceberg catalog configurations. */ -public interface IcebergCatalogWrapperProvider { +public interface IcebergCatalogConfigProvider { /** * @param properties The parameters for creating Provider which from configurations whose prefix @@ -33,8 +35,8 @@ public interface IcebergCatalogWrapperProvider { void initialize(Map properties); /** - * @param catalogName a param send by clients. - * @return the instance of IcebergCatalogWrapper. + * @param catalogName Iceberg catalog name. + * @return the configuration of Iceberg catalog. */ - IcebergCatalogWrapper getIcebergTableOps(String catalogName); + Optional getIcebergCatalogConfig(String catalogName); } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 6ff4bf2ce03..95e82aa2275 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableSet; import java.sql.Driver; import java.sql.DriverManager; -import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -104,10 +103,6 @@ public IcebergCatalogWrapper(IcebergConfig icebergConfig) { this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties(); } - public IcebergCatalogWrapper() { - this(new IcebergConfig(Collections.emptyMap())); - } - private void validateNamespace(Optional namespace) { namespace.ifPresent( n -> @@ -160,7 +155,7 @@ public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest /** * Reload hadoop configuration, this is useful when the hadoop configuration UserGroupInformation * is shared by multiple threads. UserGroupInformation#authenticationMethod was first initialized - * in KerberosClient, however, when switching to iceberg-rest thead, + * in KerberosClient, however, when switching to iceberg-rest thread, * UserGroupInformation#authenticationMethod will be reset to the default value; we need to * reinitialize it again. */ @@ -271,7 +266,7 @@ public void close() throws Exception { private void closeMySQLCatalogResource() { try { // Close thread AbandonedConnectionCleanupThread if we are using `com.mysql.cj.jdbc.Driver`, - // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thead maybe not this one. + // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thread maybe not this one. Class.forName("com.mysql.cj.jdbc.AbandonedConnectionCleanupThread") .getMethod("uncheckedShutdown") .invoke(null); diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java similarity index 81% rename from iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java index a38fd9cf302..4965f4bc132 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java @@ -21,14 +21,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils; import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,13 +40,10 @@ * *

The catalogName is iceberg_catalog */ -public class GravitinoBasedIcebergCatalogWrapperProvider - implements IcebergCatalogWrapperProvider, AutoCloseable { +public class DynamicIcebergCatalogConfigProvider + implements IcebergCatalogConfigProvider, AutoCloseable { public static final Logger LOG = - LoggerFactory.getLogger(GravitinoBasedIcebergCatalogWrapperProvider.class); - - public static final String GRAVITINO_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME = - "gravitino-based-provider"; + LoggerFactory.getLogger(DynamicIcebergCatalogConfigProvider.class); private String gravitinoMetalake; @@ -66,14 +64,19 @@ public void initialize(Map properties) { } @Override - public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { + public Optional getIcebergCatalogConfig(String catalogName) { Preconditions.checkArgument( StringUtils.isNotBlank(catalogName), "blank catalogName is illegal"); Preconditions.checkArgument( !IcebergConstants.GRAVITINO_DEFAULT_CATALOG.equals(catalogName), IcebergConstants.GRAVITINO_DEFAULT_CATALOG + " is illegal in gravitino-based-provider"); - Catalog catalog = client.loadMetalake(gravitinoMetalake).loadCatalog(catalogName); + Catalog catalog; + try { + catalog = client.loadMetalake(gravitinoMetalake).loadCatalog(catalogName); + } catch (NoSuchCatalogException e) { + return Optional.empty(); + } Preconditions.checkArgument( "lakehouse-iceberg".equals(catalog.provider()), @@ -81,7 +84,7 @@ public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { Map properties = IcebergPropertiesUtils.toIcebergCatalogProperties(catalog.properties()); - return new IcebergCatalogWrapper(new IcebergConfig(properties)); + return Optional.of(new IcebergConfig(properties)); } @VisibleForTesting diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java similarity index 78% rename from iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java index 522bca39fe3..aa7f1032134 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java @@ -24,8 +24,7 @@ import java.util.stream.Collectors; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.apache.gravitino.utils.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,11 +39,9 @@ * gravitino.iceberg-rest.catalog.hive_proxy.catalog-backend = hive * gravitino.iceberg-rest.catalog.hive_proxy.uri = thrift://{host}:{port} ... */ -public class ConfigBasedIcebergCatalogWrapperProvider implements IcebergCatalogWrapperProvider { +public class StaticIcebergCatalogConfigProvider implements IcebergCatalogConfigProvider { public static final Logger LOG = - LoggerFactory.getLogger(ConfigBasedIcebergCatalogWrapperProvider.class); - - public static final String CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME = "config-based-provider"; + LoggerFactory.getLogger(StaticIcebergCatalogConfigProvider.class); @VisibleForTesting Map catalogConfigs; @@ -68,14 +65,8 @@ public void initialize(Map properties) { } @Override - public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { - IcebergConfig icebergConfig = this.catalogConfigs.get(catalogName); - if (icebergConfig == null) { - String errorMsg = String.format("%s can not match any catalog", catalogName); - LOG.warn(errorMsg); - throw new RuntimeException(errorMsg); - } - return new IcebergCatalogWrapper(icebergConfig); + public Optional getIcebergCatalogConfig(String catalogName) { + return Optional.ofNullable(catalogConfigs.get(catalogName)); } private Optional getCatalogName(String catalogConfigKey) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java index 17342acf71f..823f42ddb16 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java @@ -21,41 +21,48 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.CredentialProviderFactory; +import org.apache.gravitino.credential.CredentialProviderManager; import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; -import org.apache.gravitino.iceberg.provider.ConfigBasedIcebergCatalogWrapperProvider; -import org.apache.gravitino.iceberg.provider.GravitinoBasedIcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.provider.DynamicIcebergCatalogConfigProvider; +import org.apache.gravitino.iceberg.provider.StaticIcebergCatalogConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class IcebergCatalogWrapperManager implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(IcebergCatalogWrapperManager.class); - private static final ImmutableMap ICEBERG_TABLE_OPS_PROVIDER_NAMES = + private static final ImmutableMap ICEBERG_CATALOG_CONFIG_PROVIDER_NAMES = ImmutableMap.of( - ConfigBasedIcebergCatalogWrapperProvider.CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME, - ConfigBasedIcebergCatalogWrapperProvider.class.getCanonicalName(), - GravitinoBasedIcebergCatalogWrapperProvider - .GRAVITINO_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME, - GravitinoBasedIcebergCatalogWrapperProvider.class.getCanonicalName()); + IcebergConstants.STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME, + StaticIcebergCatalogConfigProvider.class.getCanonicalName(), + IcebergConstants.DYNAMIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME, + DynamicIcebergCatalogConfigProvider.class.getCanonicalName()); - private final Cache icebergTableOpsCache; + private final Cache icebergCatalogWrapperCache; - private final IcebergCatalogWrapperProvider provider; + private final IcebergCatalogConfigProvider provider; + + private CredentialProviderManager credentialProviderManager; public IcebergCatalogWrapperManager(Map properties) { - this.provider = createProvider(properties); + this.credentialProviderManager = new CredentialProviderManager(); + this.provider = createIcebergCatalogConfigProvider(properties); this.provider.initialize(properties); - this.icebergTableOpsCache = + this.icebergCatalogWrapperCache = Caffeine.newBuilder() .expireAfterWrite( (new IcebergConfig(properties)) @@ -63,8 +70,10 @@ public IcebergCatalogWrapperManager(Map properties) { TimeUnit.MILLISECONDS) .removalListener( (k, v, c) -> { - LOG.info("Remove IcebergCatalogWrapper cache {}.", k); - closeIcebergTableOps((IcebergCatalogWrapper) v); + String catalogName = (String) k; + LOG.info("Remove IcebergCatalogWrapper cache {}.", catalogName); + closeIcebergCatalogWrapper((IcebergCatalogWrapper) v); + credentialProviderManager.unregisterCredentialProvider(catalogName); }) .scheduler( Scheduler.forScheduledExecutorService( @@ -72,7 +81,7 @@ public IcebergCatalogWrapperManager(Map properties) { 1, new ThreadFactoryBuilder() .setDaemon(true) - .setNameFormat("table-ops-cleaner-%d") + .setNameFormat("iceberg-catalog-wrapper-cleaner-%d") .build()))) .build(); } @@ -85,13 +94,40 @@ public IcebergCatalogWrapperManager(Map properties) { public IcebergCatalogWrapper getOps(String rawPrefix) { String catalogName = getCatalogName(rawPrefix); IcebergCatalogWrapper tableOps = - icebergTableOpsCache.get(catalogName, k -> provider.getIcebergTableOps(catalogName)); + icebergCatalogWrapperCache.get(catalogName, k -> createCatalogWrapper(catalogName)); // Reload conf to reset UserGroupInformation or icebergTableOps will always use // Simple auth. tableOps.reloadHadoopConf(); return tableOps; } + public CredentialProvider getCredentialProvider(String prefix) { + String catalogName = getCatalogName(prefix); + return credentialProviderManager.getCredentialProvider(catalogName); + } + + @VisibleForTesting + protected IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { + return new IcebergCatalogWrapper(icebergConfig); + } + + private IcebergCatalogWrapper createCatalogWrapper(String catalogName) { + Optional icebergConfig = provider.getIcebergCatalogConfig(catalogName); + if (!icebergConfig.isPresent()) { + throw new RuntimeException("Couldn't find Iceberg configuration for " + catalogName); + } + + IcebergConfig config = icebergConfig.get(); + String credentialProviderType = config.get(IcebergConfig.CREDENTIAL_PROVIDER_TYPE); + if (StringUtils.isNotBlank(credentialProviderType)) { + CredentialProvider credentialProvider = + CredentialProviderFactory.create(credentialProviderType, config.getAllConfig()); + credentialProviderManager.registerCredentialProvider(catalogName, credentialProvider); + } + + return createIcebergCatalogWrapper(icebergConfig.get()); + } + private String getCatalogName(String rawPrefix) { String prefix = shelling(rawPrefix); Preconditions.checkArgument( @@ -103,14 +139,16 @@ private String getCatalogName(String rawPrefix) { return prefix; } - private IcebergCatalogWrapperProvider createProvider(Map properties) { + private IcebergCatalogConfigProvider createIcebergCatalogConfigProvider( + Map properties) { String providerName = - (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_PROVIDER); - String className = ICEBERG_TABLE_OPS_PROVIDER_NAMES.getOrDefault(providerName, providerName); + (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_CONFIG_PROVIDER); + String className = + ICEBERG_CATALOG_CONFIG_PROVIDER_NAMES.getOrDefault(providerName, providerName); LOG.info("Load Iceberg catalog provider: {}.", className); try { Class providerClz = Class.forName(className); - return (IcebergCatalogWrapperProvider) providerClz.getDeclaredConstructor().newInstance(); + return (IcebergCatalogConfigProvider) providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } @@ -127,17 +165,17 @@ private String shelling(String rawPrefix) { } } - private void closeIcebergTableOps(IcebergCatalogWrapper ops) { + private void closeIcebergCatalogWrapper(IcebergCatalogWrapper catalogWrapper) { try { - ops.close(); + catalogWrapper.close(); } catch (Exception ex) { - LOG.warn("Close Iceberg table ops fail: {}, {}", ops, ex); + LOG.warn("Close Iceberg table catalog wrapper fail: {}, {}", catalogWrapper, ex); } } @Override public void close() throws Exception { - icebergTableOpsCache.invalidateAll(); + icebergCatalogWrapperCache.invalidateAll(); if (provider instanceof AutoCloseable) { ((AutoCloseable) provider).close(); } diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index 0c383e52063..33023343ef3 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -22,6 +22,8 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -29,6 +31,8 @@ import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HEAD; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.NotSupportedException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -37,16 +41,24 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialPropertyUtils; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.CredentialUtils; import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergObjectMapper; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ServiceUnavailableException; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.requests.ReportMetricsRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.LoadTableResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +69,9 @@ public class IcebergTableOperations { private static final Logger LOG = LoggerFactory.getLogger(IcebergTableOperations.class); + @VisibleForTesting + public static final String X_ICEBERG_ACCESS_DELEGATION = "X-Iceberg-Access-Delegation"; + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; private IcebergMetricsManager icebergMetricsManager; @@ -92,15 +107,24 @@ public Response listTable( public Response createTable( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, - CreateTableRequest createTableRequest) { + CreateTableRequest createTableRequest, + @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation) { + boolean isCredentialVending = isCredentialVending(accessDelegation); LOG.info( - "Create Iceberg table, namespace: {}, create table request: {}", + "Create Iceberg table, namespace: {}, create table request: {}, accessDelegation: {}, isCredentialVending: {}", namespace, - createTableRequest); - return IcebergRestUtils.ok( + createTableRequest, + accessDelegation, + isCredentialVending); + LoadTableResponse loadTableResponse = icebergCatalogWrapperManager .getOps(prefix) - .createTable(RESTUtil.decodeNamespace(namespace), createTableRequest)); + .createTable(RESTUtil.decodeNamespace(namespace), createTableRequest); + if (isCredentialVending) { + return IcebergRestUtils.ok(injectCredentialConfig(prefix, loadTableResponse)); + } else { + return IcebergRestUtils.ok(loadTableResponse); + } } @POST @@ -162,12 +186,26 @@ public Response loadTable( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, - @DefaultValue("all") @QueryParam("snapshots") String snapshots) { + @DefaultValue("all") @QueryParam("snapshots") String snapshots, + @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation) { + boolean isCredentialVending = isCredentialVending(accessDelegation); + LOG.info( + "Load iceberg table, namespace: {}, table: {}, access delegation: {}, " + + "credential vending: {}", + namespace, + table, + accessDelegation, + isCredentialVending); // todo support snapshots TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); - return IcebergRestUtils.ok( - icebergCatalogWrapperManager.getOps(prefix).loadTable(tableIdentifier)); + LoadTableResponse loadTableResponse = + icebergCatalogWrapperManager.getOps(prefix).loadTable(tableIdentifier); + if (isCredentialVending) { + return IcebergRestUtils.ok(injectCredentialConfig(prefix, loadTableResponse)); + } else { + return IcebergRestUtils.ok(loadTableResponse); + } } @HEAD @@ -210,4 +248,49 @@ private String SerializeUpdateTableRequest(UpdateTableRequest updateTableRequest return updateTableRequest.toString(); } } + + private LoadTableResponse injectCredentialConfig( + String prefix, LoadTableResponse loadTableResponse) { + CredentialProvider credentialProvider = + icebergCatalogWrapperManager.getCredentialProvider(prefix); + if (credentialProvider == null) { + throw new NotSupportedException( + "Doesn't support credential vending, please add " + + CredentialConstants.CREDENTIAL_PROVIDER_TYPE + + " to the catalog configurations"); + } + Credential credential = + CredentialUtils.vendCredential( + credentialProvider, loadTableResponse.tableMetadata().location()); + if (credential == null) { + throw new ServiceUnavailableException( + "Couldn't generate credential for %s", credentialProvider.credentialType()); + } + Map credentialConfig = CredentialPropertyUtils.toIcebergProperties(credential); + return LoadTableResponse.builder() + .withTableMetadata(loadTableResponse.tableMetadata()) + .addAllConfig(loadTableResponse.config()) + .addAllConfig(credentialConfig) + .build(); + } + + private boolean isCredentialVending(String accessDelegation) { + if (StringUtils.isBlank(accessDelegation)) { + return false; + } + if ("vended-credentials".equalsIgnoreCase(accessDelegation)) { + return true; + } + if ("remote-signing".equalsIgnoreCase(accessDelegation)) { + throw new UnsupportedOperationException( + "Gravitino IcebergRESTServer doesn't support remote signing"); + } else { + throw new IllegalArgumentException( + X_ICEBERG_ACCESS_DELEGATION + + ": " + + accessDelegation + + " is illegal, Iceberg REST spec supports:[vended-credentials,remote-signing], " + + "Gravitino Iceberg REST server supports: vended-credentials"); + } + } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java similarity index 87% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java index 8acac4ffd6b..f9ffbb42747 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -public class TestGravitinoBasedIcebergCatalogWrapperProvider { +public class TestDynamicIcebergCatalogWrapperProvider { @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -71,14 +71,15 @@ public void testValidIcebergTableOps() { } }); - GravitinoBasedIcebergCatalogWrapperProvider provider = - new GravitinoBasedIcebergCatalogWrapperProvider(); + DynamicIcebergCatalogConfigProvider provider = new DynamicIcebergCatalogConfigProvider(); GravitinoAdminClient client = Mockito.mock(GravitinoAdminClient.class); Mockito.when(client.loadMetalake(Mockito.any())).thenReturn(gravitinoMetalake); provider.setClient(client); - IcebergCatalogWrapper hiveOps = provider.getIcebergTableOps(hiveCatalogName); - IcebergCatalogWrapper jdbcOps = provider.getIcebergTableOps(jdbcCatalogName); + IcebergCatalogWrapper hiveOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(hiveCatalogName).get()); + IcebergCatalogWrapper jdbcOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(jdbcCatalogName).get()); Assertions.assertEquals(hiveCatalogName, hiveOps.getCatalog().name()); Assertions.assertEquals(jdbcCatalogName, jdbcOps.getCatalog().name()); @@ -101,16 +102,15 @@ public void testInvalidIcebergTableOps() { GravitinoAdminClient client = Mockito.mock(GravitinoAdminClient.class); Mockito.when(client.loadMetalake(Mockito.any())).thenReturn(gravitinoMetalake); - GravitinoBasedIcebergCatalogWrapperProvider provider = - new GravitinoBasedIcebergCatalogWrapperProvider(); + DynamicIcebergCatalogConfigProvider provider = new DynamicIcebergCatalogConfigProvider(); provider.setClient(client); Assertions.assertThrowsExactly( - IllegalArgumentException.class, () -> provider.getIcebergTableOps(invalidCatalogName)); + IllegalArgumentException.class, () -> provider.getIcebergCatalogConfig(invalidCatalogName)); Assertions.assertThrowsExactly( - IllegalArgumentException.class, () -> provider.getIcebergTableOps("")); + IllegalArgumentException.class, () -> provider.getIcebergCatalogConfig("")); Assertions.assertThrowsExactly( IllegalArgumentException.class, - () -> provider.getIcebergTableOps(IcebergConstants.GRAVITINO_DEFAULT_CATALOG)); + () -> provider.getIcebergCatalogConfig(IcebergConstants.GRAVITINO_DEFAULT_CATALOG)); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java similarity index 86% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java index 99e83f2e41d..69f5b5ad257 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import java.util.Map; +import java.util.Optional; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; @@ -31,7 +32,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -public class TestConfigBasedIcebergCatalogWrapperProvider { +public class TestStaticIcebergCatalogWrapperProvider { + @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -58,16 +60,18 @@ public void testValidIcebergTableOps() { config.put("catalog-backend", "memory"); config.put("warehouse", "/tmp/"); - ConfigBasedIcebergCatalogWrapperProvider provider = - new ConfigBasedIcebergCatalogWrapperProvider(); + StaticIcebergCatalogConfigProvider provider = new StaticIcebergCatalogConfigProvider(); provider.initialize(config); IcebergConfig hiveIcebergConfig = provider.catalogConfigs.get(hiveCatalogName); IcebergConfig jdbcIcebergConfig = provider.catalogConfigs.get(jdbcCatalogName); IcebergConfig defaultIcebergConfig = provider.catalogConfigs.get(defaultCatalogName); - IcebergCatalogWrapper hiveOps = provider.getIcebergTableOps(hiveCatalogName); - IcebergCatalogWrapper jdbcOps = provider.getIcebergTableOps(jdbcCatalogName); - IcebergCatalogWrapper defaultOps = provider.getIcebergTableOps(defaultCatalogName); + IcebergCatalogWrapper hiveOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(hiveCatalogName).get()); + IcebergCatalogWrapper jdbcOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(jdbcCatalogName).get()); + IcebergCatalogWrapper defaultOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(defaultCatalogName).get()); Assertions.assertEquals( hiveCatalogName, hiveIcebergConfig.get(IcebergConfig.CATALOG_BACKEND_NAME)); @@ -102,11 +106,10 @@ public void testValidIcebergTableOps() { @ParameterizedTest @ValueSource(strings = {"", "not_match"}) public void testInvalidIcebergTableOps(String catalogName) { - ConfigBasedIcebergCatalogWrapperProvider provider = - new ConfigBasedIcebergCatalogWrapperProvider(); + StaticIcebergCatalogConfigProvider provider = new StaticIcebergCatalogConfigProvider(); provider.initialize(Maps.newHashMap()); - Assertions.assertThrowsExactly( - RuntimeException.class, () -> provider.getIcebergTableOps(catalogName)); + Optional config = provider.getIcebergCatalogConfig(catalogName); + Assertions.assertEquals(Optional.empty(), config); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java new file mode 100644 index 00000000000..6b1e4c08710 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.extension; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; + +public class DummyCredentialProvider implements CredentialProvider { + public static final String DUMMY_CREDENTIAL_TYPE = "iceberg-rest-dummy-test"; + + public static class SimpleCredential implements Credential { + @Override + public String credentialType() { + return DUMMY_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return new HashMap<>(); + } + } + + @Override + public void initialize(Map properties) {} + + @Override + public String credentialType() { + return DUMMY_CREDENTIAL_TYPE; + } + + @Nullable + @Override + public Credential getCredential(CredentialContext context) { + return new SimpleCredential(); + } + + @Override + public void close() throws IOException {} +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java deleted file mode 100644 index 222391bcc04..00000000000 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.iceberg.service.rest; - -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.provider.ConfigBasedIcebergCatalogWrapperProvider; - -public class ConfigBasedIcebergCatalogWrapperProviderForTest - extends ConfigBasedIcebergCatalogWrapperProvider { - @Override - public IcebergCatalogWrapper getIcebergTableOps(String prefix) { - return new IcebergCatalogWrapperForTest(); - } -} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java index 69c0a50e409..f6326dd229e 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.iceberg.service.rest; +import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -30,7 +31,12 @@ import org.apache.iceberg.types.Types.StringType; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +// Used to override registerTable public class IcebergCatalogWrapperForTest extends IcebergCatalogWrapper { + public IcebergCatalogWrapperForTest(IcebergConfig icebergConfig) { + super(icebergConfig); + } + @Override public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest request) { if (request.name().contains("fail")) { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java new file mode 100644 index 00000000000..7d359926a85 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.rest; + +import java.util.Map; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; + +// Provide a custom catalogWrapper to do test like `registerTable` +public class IcebergCatalogWrapperManagerForTest extends IcebergCatalogWrapperManager { + public IcebergCatalogWrapperManagerForTest(Map properties) { + super(properties); + } + + @Override + public IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { + return new IcebergCatalogWrapperForTest(icebergConfig); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index 4fc645132e1..1a085a251d9 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -24,10 +24,13 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.provider.StaticIcebergCatalogConfigProvider; import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergExceptionMapper; import org.apache.gravitino.iceberg.service.IcebergObjectMapperProvider; +import org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; @@ -74,12 +77,17 @@ public static ResourceConfig getIcebergResourceConfig(Class c, boolean bindIcebe if (bindIcebergTableOps) { Map catalogConf = Maps.newHashMap(); - catalogConf.put(String.format("catalog.%s.catalog-backend-name", PREFIX), PREFIX); + String catalogConfigPrefix = "catalog." + PREFIX; catalogConf.put( - IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER, - ConfigBasedIcebergCatalogWrapperProviderForTest.class.getName()); + IcebergConstants.ICEBERG_REST_CATALOG_CONFIG_PROVIDER, + StaticIcebergCatalogConfigProvider.class.getName()); + catalogConf.put(String.format("%s.catalog-backend-name", catalogConfigPrefix), PREFIX); + catalogConf.put( + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE); + // used to override register table interface IcebergCatalogWrapperManager icebergCatalogWrapperManager = - new IcebergCatalogWrapperManager(catalogConf); + new IcebergCatalogWrapperManagerForTest(catalogConf); IcebergMetricsManager icebergMetricsManager = new IcebergMetricsManager(new IcebergConfig()); resourceConfig.register( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java index 6037302b8b2..809a4ff2cd5 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java @@ -29,6 +29,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; @@ -55,6 +57,12 @@ public class TestIcebergTableOperations extends TestIcebergNamespaceOperations { + private static final Schema tableSchema = + new Schema(NestedField.of(1, false, "foo_string", StringType.get())); + + private static final Schema newTableSchema = + new Schema(NestedField.of(2, false, "foo_string1", StringType.get())); + @Override protected Application configure() { ResourceConfig resourceConfig = @@ -66,11 +74,163 @@ protected Application configure() { return resourceConfig; } - private static final Schema tableSchema = - new Schema(NestedField.of(1, false, "foo_string", StringType.get())); + @Test + void testCreateTable() { + verifyCreateTableFail("create_foo1", 404); - private static final Schema newTableSchema = - new Schema(NestedField.of(2, false, "foo_string1", StringType.get())); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + verifyCreateTableSucc("create_foo1"); + + verifyCreateTableFail("create_foo1", 409); + verifyCreateTableFail("", 400); + } + + @Test + void testLoadTable() { + verifyLoadTableFail("load_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("load_foo1"); + verifyLoadTableSucc("load_foo1"); + + verifyLoadTableFail("load_foo2", 404); + } + + @Test + void testDropTable() { + verifyDropTableFail("drop_foo1", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyDropTableFail("drop_foo1", 404); + + verifyCreateTableSucc("drop_foo1"); + verifyDropTableSucc("drop_foo1"); + verifyLoadTableFail("drop_foo1", 404); + } + + @Test + void testUpdateTable() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("update_foo1"); + TableMetadata metadata = getTableMeta("update_foo1"); + verifyUpdateSucc("update_foo1", metadata); + + verifyDropTableSucc("update_foo1"); + verifyUpdateTableFail("update_foo1", 404, metadata); + + verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyUpdateTableFail("update_foo1", 404, metadata); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListTables(String prefix) { + setUrlPathWithPrefix(prefix); + verifyListTableFail(404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("list_foo1"); + verifyCreateTableSucc("list_foo2"); + verifyListTableSucc(ImmutableSet.of("list_foo1", "list_foo2")); + } + + @Test + void testTableExits() { + verifyTableExistsStatusCode("exists_foo2", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyTableExistsStatusCode("exists_foo2", 404); + + verifyCreateTableSucc("exists_foo1"); + verifyTableExistsStatusCode("exists_foo1", 200); + verifyLoadTableSucc("exists_foo1"); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); + // namespace not exits + verifyRenameTableFail("rename_foo1", "rename_foo3", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("rename_foo1"); + // rename + verifyRenameTableSucc("rename_foo1", "rename_foo2"); + verifyLoadTableFail("rename_foo1", 404); + verifyLoadTableSucc("rename_foo2"); + + // source table not exists + verifyRenameTableFail("rename_foo1", "rename_foo3", 404); + + // dest table exists + verifyCreateTableSucc("rename_foo3"); + verifyRenameTableFail("rename_foo2", "rename_foo3", 409); + } + + @Test + void testReportTableMetrics() { + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("metrics_foo1"); + + ImmutableCommitMetricsResult commitMetrics = ImmutableCommitMetricsResult.builder().build(); + CommitReport commitReport = + ImmutableCommitReport.builder() + .tableName("metrics_foo1") + .snapshotId(-1) + .sequenceNumber(-1) + .operation("append") + .commitMetrics(commitMetrics) + .build(); + ReportMetricsRequest request = ReportMetricsRequest.of(commitReport); + Response response = + getReportMetricsClientBuilder("metrics_foo1") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + @Test + void testCreateTableWithCredentialVending() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + // create the table without credential vending + Response response = doCreateTable("create_without_credential_vending"); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertTrue(!loadTableResponse.config().containsKey(Credential.CREDENTIAL_TYPE)); + + // create the table with credential vending + String tableName = "create_with_credential_vending"; + response = doCreateTableWithCredentialVending(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertEquals( + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE, + loadTableResponse.config().get(Credential.CREDENTIAL_TYPE)); + + // load the table without credential vending + response = doLoadTable(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertTrue(!loadTableResponse.config().containsKey(Credential.CREDENTIAL_TYPE)); + + // load the table with credential vending + response = doLoadTableWithCredentialVending(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertEquals( + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE, + loadTableResponse.config().get(Credential.CREDENTIAL_TYPE)); + } + + private Response doCreateTableWithCredentialVending(String name) { + CreateTableRequest createTableRequest = + CreateTableRequest.builder().withName(name).withSchema(tableSchema).build(); + return getTableClientBuilder() + .header(IcebergTableOperations.X_ICEBERG_ACCESS_DELEGATION, "vended-credentials") + .post(Entity.entity(createTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } private Response doCreateTable(String name) { CreateTableRequest createTableRequest = @@ -103,6 +263,12 @@ private Response doTableExists(String name) { return getTableClientBuilder(Optional.of(name)).head(); } + private Response doLoadTableWithCredentialVending(String name) { + return getTableClientBuilder(Optional.of(name)) + .header(IcebergTableOperations.X_ICEBERG_ACCESS_DELEGATION, "vended-credentials") + .get(); + } + private Response doLoadTable(String name) { return getTableClientBuilder(Optional.of(name)).get(); } @@ -116,6 +282,12 @@ private Response doUpdateTable(String name, TableMetadata base) { .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); } + private TableMetadata getTableMeta(String tableName) { + Response response = doLoadTable(tableName); + LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); + return loadTableResponse.tableMetadata(); + } + private void verifyUpdateTableFail(String name, int status, TableMetadata base) { Response response = doUpdateTable(name, base); Assertions.assertEquals(status, response.getStatus()); @@ -204,126 +376,4 @@ private void verifyCreateTableFail(String name, int status) { Response response = doCreateTable(name); Assertions.assertEquals(status, response.getStatus()); } - - @Test - void testCreateTable() { - verifyCreateTableFail("create_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - - verifyCreateTableSucc("create_foo1"); - - verifyCreateTableFail("create_foo1", 409); - verifyCreateTableFail("", 400); - } - - @Test - void testLoadTable() { - verifyLoadTableFail("load_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("load_foo1"); - verifyLoadTableSucc("load_foo1"); - - verifyLoadTableFail("load_foo2", 404); - } - - @Test - void testDropTable() { - verifyDropTableFail("drop_foo1", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyDropTableFail("drop_foo1", 404); - - verifyCreateTableSucc("drop_foo1"); - verifyDropTableSucc("drop_foo1"); - verifyLoadTableFail("drop_foo1", 404); - } - - private TableMetadata getTableMeta(String tableName) { - Response response = doLoadTable(tableName); - LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); - return loadTableResponse.tableMetadata(); - } - - @Test - void testUpdateTable() { - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("update_foo1"); - TableMetadata metadata = getTableMeta("update_foo1"); - verifyUpdateSucc("update_foo1", metadata); - - verifyDropTableSucc("update_foo1"); - verifyUpdateTableFail("update_foo1", 404, metadata); - - verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyUpdateTableFail("update_foo1", 404, metadata); - } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testListTables(String prefix) { - setUrlPathWithPrefix(prefix); - verifyListTableFail(404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("list_foo1"); - verifyCreateTableSucc("list_foo2"); - verifyListTableSucc(ImmutableSet.of("list_foo1", "list_foo2")); - } - - @Test - void testTableExits() { - verifyTableExistsStatusCode("exists_foo2", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyTableExistsStatusCode("exists_foo2", 404); - - verifyCreateTableSucc("exists_foo1"); - verifyTableExistsStatusCode("exists_foo1", 200); - verifyLoadTableSucc("exists_foo1"); - } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testRenameTable(String prefix) { - setUrlPathWithPrefix(prefix); - // namespace not exits - verifyRenameTableFail("rename_foo1", "rename_foo3", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("rename_foo1"); - // rename - verifyRenameTableSucc("rename_foo1", "rename_foo2"); - verifyLoadTableFail("rename_foo1", 404); - verifyLoadTableSucc("rename_foo2"); - - // source table not exists - verifyRenameTableFail("rename_foo1", "rename_foo3", 404); - - // dest table exists - verifyCreateTableSucc("rename_foo3"); - verifyRenameTableFail("rename_foo2", "rename_foo3", 409); - } - - @Test - void testReportTableMetrics() { - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("metrics_foo1"); - - ImmutableCommitMetricsResult commitMetrics = ImmutableCommitMetricsResult.builder().build(); - CommitReport commitReport = - ImmutableCommitReport.builder() - .tableName("metrics_foo1") - .snapshotId(-1) - .sequenceNumber(-1) - .operation("append") - .commitMetrics(commitMetrics) - .build(); - ReportMetricsRequest request = ReportMetricsRequest.of(commitReport); - Response response = - getReportMetricsClientBuilder("metrics_foo1") - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); - - Assertions.assertEquals(Status.NO_CONTENT.getStatusCode(), response.getStatus()); - } } diff --git a/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider new file mode 100644 index 00000000000..25a4f2d46b1 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider