[#3462] improvement(hadoop-catalog): Support different level(catalog,…

… schema,fileset) of authentication for hadoop catalog. (#3852) ### What changes were proposed in this pull request? Support set authentication for schema and fileset level for Hadoop catalog. ### Why are the changes needed? The Hadoo catalog may require fine-grained access control and we can check the authentication status in the catalog, schema, or fileset level. For instance, when considering Kerberos authentication, we can authenticate filesets by catalog level, schema level, and file level, and fileset level authentication configuration has the highest priority. Fix: #3462 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? IT
apache · Jul 23, 2024 · d7a5808 · d7a5808
1 parent 71e6651
commit d7a5808
Show file tree

Hide file tree

Showing 19 changed files with 930 additions and 128 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -163,7 +163,7 @@ allprojects {
       // Default use MiniGravitino to run integration tests
       param.environment("GRAVITINO_ROOT_DIR", project.rootDir.path)
       param.environment("IT_PROJECT_DIR", project.buildDir.path)
-      param.environment("HADOOP_USER_NAME", "datastrato")
+      param.environment("HADOOP_USER_NAME", "anonymous")
       param.environment("HADOOP_HOME", "/tmp")
       param.environment("PROJECT_VERSION", project.version)
 

diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalog.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalog.java
@@ -19,12 +19,9 @@
 package org.apache.gravitino.catalog.hadoop;
 
 import java.util.Map;
-import java.util.Optional;
-import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import org.apache.gravitino.connector.BaseCatalog;
 import org.apache.gravitino.connector.CatalogOperations;
 import org.apache.gravitino.connector.PropertiesMetadata;
-import org.apache.gravitino.connector.ProxyPlugin;
 import org.apache.gravitino.connector.capability.Capability;
 
 /**
@@ -50,7 +47,7 @@ public String shortName() {
 
   @Override
   protected CatalogOperations newOps(Map<String, String> config) {
-    HadoopCatalogOperations ops = new HadoopCatalogOperations();
+    CatalogOperations ops = new SecureHadoopCatalogOperations();
     return ops;
   }
 
@@ -59,15 +56,6 @@ protected Capability newCapability() {
     return new HadoopCatalogCapability();
   }
 
-  @Override
-  protected Optional<ProxyPlugin> newProxyPlugin(Map<String, String> config) {
-    boolean impersonationEnabled = new KerberosConfig(config).isImpersonationEnabled();
-    if (!impersonationEnabled) {
-      return Optional.empty();
-    }
-    return Optional.of(new HadoopProxyPlugin());
-  }
-
   @Override
   public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException {
     return CATALOG_PROPERTIES_META;

diff --git a/...log-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/...log-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java
@@ -19,17 +19,14 @@
 package org.apache.gravitino.catalog.hadoop;
 
 import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
-import java.io.File;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -43,12 +40,9 @@
 import org.apache.gravitino.Schema;
 import org.apache.gravitino.SchemaChange;
 import org.apache.gravitino.StringIdentifier;
-import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
-import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosClient;
 import org.apache.gravitino.connector.CatalogInfo;
 import org.apache.gravitino.connector.CatalogOperations;
 import org.apache.gravitino.connector.HasPropertyMetadata;
-import org.apache.gravitino.connector.ProxyPlugin;
 import org.apache.gravitino.connector.SupportsSchemas;
 import org.apache.gravitino.exceptions.AlreadyExistsException;
 import org.apache.gravitino.exceptions.FilesetAlreadyExistsException;
@@ -68,8 +62,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -90,11 +82,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem
 
   private Map<String, String> conf;
 
-  @SuppressWarnings("unused")
-  private ProxyPlugin proxyPlugin;
-
-  private String kerberosRealm;
-
   private CatalogInfo catalogInfo;
 
   HadoopCatalogOperations(EntityStore store) {
@@ -105,8 +92,20 @@ public HadoopCatalogOperations() {
     this(GravitinoEnv.getInstance().entityStore());
   }
 
-  public String getKerberosRealm() {
-    return kerberosRealm;
+  public EntityStore getStore() {
+    return store;
+  }
+
+  public CatalogInfo getCatalogInfo() {
+    return catalogInfo;
+  }
+
+  public Configuration getHadoopConf() {
+    return hadoopConf;
+  }
+
+  public Map<String, String> getConf() {
+    return conf;
   }
 
   @Override
@@ -134,32 +133,12 @@ public void initialize(
                 .getOrDefault(config, HadoopCatalogPropertiesMetadata.LOCATION);
     conf.forEach(hadoopConf::set);
 
-    initAuthentication(conf, hadoopConf);
     this.catalogStorageLocation =
         StringUtils.isNotBlank(catalogLocation)
             ? Optional.of(catalogLocation).map(Path::new)
             : Optional.empty();
   }
 
-  private void initAuthentication(Map<String, String> conf, Configuration hadoopConf) {
-    AuthenticationConfig config = new AuthenticationConfig(conf);
-    String authType = config.getAuthType();
-
-    if (StringUtils.equalsIgnoreCase(authType, AuthenticationMethod.KERBEROS.name())) {
-      hadoopConf.set(
-          HADOOP_SECURITY_AUTHENTICATION,
-          AuthenticationMethod.KERBEROS.name().toLowerCase(Locale.ROOT));
-      UserGroupInformation.setConfiguration(hadoopConf);
-      try {
-        KerberosClient kerberosClient = new KerberosClient(conf, hadoopConf);
-        File keytabFile = kerberosClient.saveKeyTabFileFromUri(catalogInfo.id());
-        this.kerberosRealm = kerberosClient.login(keytabFile.getAbsolutePath());
-      } catch (IOException e) {
-        throw new RuntimeException("Failed to login with Kerberos", e);
-      }
-    }
-  }
-
   @Override
   public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException {
     try {
@@ -278,9 +257,9 @@ public Fileset createFileset(
             .withNamespace(ident.namespace())
             .withComment(comment)
             .withFilesetType(type)
-            // Store the storageLocation to the store. If the "storageLocation" is null for
-            // managed fileset, Gravitino will get and store the location based on the
-            // catalog/schema's location and store it to the store.
+            // Store the storageLocation to the store. If the "storageLocation" is null for managed
+            // fileset, Gravitino will get and store the location based on the catalog/schema's
+            // location and store it to the store.
             .withStorageLocation(filesetPath.toString())
             .withProperties(properties)
             .withAuditInfo(
@@ -664,8 +643,4 @@ static Path formalizePath(Path path, Configuration configuration) throws IOExcep
     FileSystem defaultFs = FileSystem.get(configuration);
     return path.makeQualified(defaultFs.getUri(), defaultFs.getWorkingDirectory());
   }
-
-  void setProxyPlugin(HadoopProxyPlugin hadoopProxyPlugin) {
-    this.proxyPlugin = hadoopProxyPlugin;
-  }
 }
diff --git a/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java b/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
@@ -18,15 +18,20 @@
  */
 package org.apache.gravitino.catalog.hadoop;
 
-import java.util.Collections;
+import com.google.common.collect.ImmutableMap;
 import java.util.Map;
+import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
+import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import org.apache.gravitino.connector.BasePropertiesMetadata;
 import org.apache.gravitino.connector.PropertyEntry;
 
 public class HadoopFilesetPropertiesMetadata extends BasePropertiesMetadata {
 
   @Override
   protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
-    return Collections.emptyMap();
+    ImmutableMap.Builder<String, PropertyEntry<?>> builder = ImmutableMap.builder();
+    builder.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES);
+    builder.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES);
+    return builder.build();
   }
 }
diff --git a/...s/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopProxyPlugin.java b/...s/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopProxyPlugin.java
@@ -31,9 +31,10 @@
 import org.apache.gravitino.utils.Executable;
 import org.apache.hadoop.security.UserGroupInformation;
 
+@Deprecated
 public class HadoopProxyPlugin implements ProxyPlugin {
-  private HadoopCatalogOperations ops;
-  private UserGroupInformation realUser;
+  private SecureHadoopCatalogOperations ops;
+  private final UserGroupInformation realUser;
 
   public HadoopProxyPlugin() {
     try {
@@ -82,7 +83,6 @@ public Object doAs(
 
   @Override
   public void bindCatalogOperation(CatalogOperations ops) {
-    this.ops = ((HadoopCatalogOperations) ops);
-    this.ops.setProxyPlugin(this);
+    this.ops = ((SecureHadoopCatalogOperations) ops);
   }
 }
diff --git a/...oop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java b/...oop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java
@@ -20,6 +20,8 @@
 
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;
+import org.apache.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
+import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import org.apache.gravitino.connector.BasePropertiesMetadata;
 import org.apache.gravitino.connector.PropertyEntry;
 
@@ -45,6 +47,8 @@ public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata {
                   true /* immutable */,
                   null,
                   false /* hidden */))
+          .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES)
+          .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES)
           .build();
 
   @Override