Skip to content

Commit

Permalink
[apache#5779] feat(core): Wildcard properties meta
Browse files Browse the repository at this point in the history
  • Loading branch information
@xunliu
xunliu committed Dec 7, 2024
1 parent c0bdd46 commit 9ef2d81
Show file tree
Hide file tree
Showing 11 changed files with 796 additions and 13 deletions.
2 changes: 1 addition & 1 deletion ...talog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata {
DEFAULT_LIST_ALL_TABLES,
false /* hidden */,
false /* reserved */))
.putAll(AuthorizationPropertiesMeta.RANGER_AUTHORIZATION_PROPERTY_ENTRIES)
.putAll(AuthorizationPropertiesMeta.AUTHORIZATION_PROPERTY_ENTRIES)
.putAll(CLIENT_PROPERTIES_METADATA.propertyEntries())
.build();

Expand Down
20 changes: 19 additions & 1 deletion ...talog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ void testPropertyMeta() {
Map<String, PropertyEntry<?>> propertyEntryMap =
HIVE_PROPERTIES_METADATA.catalogPropertiesMetadata().propertyEntries();

Assertions.assertEquals(21, propertyEntryMap.size());
Assertions.assertEquals(29, propertyEntryMap.size());
Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS));
Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE));
Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL));
Expand All @@ -93,6 +93,24 @@ void testPropertyMeta() {
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_PASSWORD));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_SERVICE_NAME));
Assertions.assertTrue(
propertyEntryMap.containsKey(
AuthorizationPropertiesMeta.getInstance().wildcardNodePropertyKey()));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_CATALOG_PROVIDER));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_CATALOG_PROVIDER));
Assertions.assertTrue(propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_PROVIDER));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_RANGER_ADMIN_URL));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_RANGER_AUTH_TYPES));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_RANGER_USERNAME));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_RANGER_PASSWORD));
Assertions.assertTrue(
propertyEntryMap.containsKey(AuthorizationPropertiesMeta.CHAIN_RANGER_SERVICE_NAME));

Assertions.assertTrue(propertyEntryMap.get(METASTORE_URIS).isRequired());
Assertions.assertFalse(propertyEntryMap.get(Catalog.PROPERTY_PACKAGE).isRequired());
Expand Down
3 changes: 3 additions & 0 deletions core/src/main/java/org/apache/gravitino/catalog/PropertiesMetadataHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.stream.Collectors;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.connector.PropertyEntry;
import org.apache.gravitino.connector.WildcardPropertiesMeta;

/** This class contains helper methods for properties metadata. */
public class PropertiesMetadataHelpers {
Expand Down Expand Up @@ -67,6 +68,8 @@ public static void validatePropertyForCreate(
"Properties are required and must be set: %s",
absentProperties);

WildcardPropertiesMeta.validate(propertiesMetadata, properties);

// use decode function to validate the property values
for (Map.Entry<String, String> entry : properties.entrySet()) {
String key = entry.getKey();
Expand Down
175 changes: 168 additions & 7 deletions core/src/main/java/org/apache/gravitino/connector/AuthorizationPropertiesMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,166 @@
import com.google.common.collect.ImmutableMap;
import java.util.Map;

public class AuthorizationPropertiesMeta {
public class AuthorizationPropertiesMeta extends BasePropertiesMetadata
implements WildcardPropertiesMeta {
private static volatile AuthorizationPropertiesMeta instance = null;

public static synchronized AuthorizationPropertiesMeta getInstance() {
if (instance == null) {
synchronized (AuthorizationPropertiesMeta.class) {
if (instance == null) {
instance = new AuthorizationPropertiesMeta();
}
}
}
return instance;
}

/** Ranger admin web URIs */
public static final String RANGER_ADMIN_URL = "authorization.ranger.admin.url";
private static final String RANGER_ADMIN_URL_KEY = "ranger.admin.url";

public static final String getRangerAdminUrlKey() {
return RANGER_ADMIN_URL_KEY;
}

public static final String RANGER_ADMIN_URL =
AuthorizationPropertiesMeta.getInstance().generateFirstNodePropertyKey(RANGER_ADMIN_URL_KEY);
/** Ranger authentication type kerberos or simple */
public static final String RANGER_AUTH_TYPE = "authorization.ranger.auth.type";
private static final String RANGER_AUTH_TYPE_KEY = "ranger.auth.type";

public static final String getRangerAuthTypeKey() {
return RANGER_AUTH_TYPE_KEY;
}

public static final String RANGER_AUTH_TYPE =
AuthorizationPropertiesMeta.getInstance().generateFirstNodePropertyKey(RANGER_AUTH_TYPE_KEY);
/**
* Ranger admin web login username(auth_type=simple), or kerberos principal(auth_type=kerberos)
*/
public static final String RANGER_USERNAME = "authorization.ranger.username";
private static final String RANGER_USERNAME_KEY = "ranger.username";

public static final String getRangerUsernameKey() {
return RANGER_USERNAME_KEY;
}

public static final String RANGER_USERNAME =
AuthorizationPropertiesMeta.getInstance().generateFirstNodePropertyKey(RANGER_USERNAME_KEY);
/**
* Ranger admin web login user password(auth_type=simple), or path of the keytab
* file(auth_type=kerberos)
*/
public static final String RANGER_PASSWORD = "authorization.ranger.password";
private static final String RANGER_PASSWORD_KEY = "ranger.password";

public static final String getRangerPasswordKey() {
return RANGER_PASSWORD_KEY;
}

public static final String RANGER_PASSWORD =
AuthorizationPropertiesMeta.getInstance().generateFirstNodePropertyKey(RANGER_PASSWORD_KEY);

/** Ranger service name */
public static final String RANGER_SERVICE_NAME = "authorization.ranger.service.name";
private static final String RANGER_SERVICE_NAME_KEY = "ranger.service.name";

public static final String getRangerServiceNameKey() {
return RANGER_SERVICE_NAME_KEY;
}

public static final String RANGER_SERVICE_NAME =
AuthorizationPropertiesMeta.getInstance()
.generateFirstNodePropertyKey(RANGER_SERVICE_NAME_KEY);

/** Chain authorization plugin provider */
private static final String CHAIN_CATALOG_PROVIDER_KEY = "catalog-provider";

public static final String getChainCatalogProviderKey() {
return CHAIN_CATALOG_PROVIDER_KEY;
}

public static final String CHAIN_CATALOG_PROVIDER =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, CHAIN_CATALOG_PROVIDER_KEY);

/** Chain authorization plugin provider */
private static final String CHAIN_PROVIDER_KEY = "provider";

public static final Map<String, PropertyEntry<?>> RANGER_AUTHORIZATION_PROPERTY_ENTRIES =
public static final String getChainProviderKey() {
return CHAIN_PROVIDER_KEY;
}

public static final String CHAIN_PROVIDER =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, CHAIN_PROVIDER_KEY);
/** Chain authorization Ranger admin web URIs */
public static final String CHAIN_RANGER_ADMIN_URL =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, RANGER_ADMIN_URL_KEY);
/** Chain authorization Ranger authentication type kerberos or simple */
public static final String CHAIN_RANGER_AUTH_TYPES =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, RANGER_AUTH_TYPE_KEY);
/** Chain authorization Ranger username */
public static final String CHAIN_RANGER_USERNAME =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, RANGER_USERNAME_KEY);
/**
* Chain authorization Ranger admin web login user password(auth_type=simple), or path of the
* keytab file(auth_type=kerberos)
*/
public static final String CHAIN_RANGER_PASSWORD =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, RANGER_PASSWORD_KEY);
/** Chain authorization Ranger service name */
public static final String CHAIN_RANGER_SERVICE_NAME =
AuthorizationPropertiesMeta.getInstance()
.getPropertyValue(Constants.WILDCARD, RANGER_SERVICE_NAME_KEY);

public static String chainKeyToPluginKey(String chainKey, String plugin) {
return chainKey.replace(
String.format(
"%s.%s", AuthorizationPropertiesMeta.getInstance().wildcardNodePropertyKey(), plugin),
AuthorizationPropertiesMeta.getInstance().firstNodeName());
}

public static final Map<String, PropertyEntry<?>> AUTHORIZATION_PROPERTY_ENTRIES =
ImmutableMap.<String, PropertyEntry<?>>builder()
.put(
AuthorizationPropertiesMeta.getInstance().wildcardNodePropertyKey(),
PropertyEntry.wildcardPropertyEntry(
AuthorizationPropertiesMeta.getInstance().wildcardNodePropertyKey(),
"The Chain authorization plugins"))
.put(
CHAIN_CATALOG_PROVIDER,
PropertyEntry.wildcardPropertyEntry(
CHAIN_PROVIDER, "The Chain sub entity catalog provider"))
.put(
CHAIN_PROVIDER,
PropertyEntry.wildcardPropertyEntry(
CHAIN_PROVIDER, "The Chain sub entity authorization plugin provider"))
.put(
CHAIN_RANGER_SERVICE_NAME,
PropertyEntry.wildcardPropertyEntry(
CHAIN_RANGER_SERVICE_NAME,
"The Chain sub entity authorization Ranger service name"))
.put(
CHAIN_RANGER_ADMIN_URL,
PropertyEntry.wildcardPropertyEntry(
CHAIN_RANGER_ADMIN_URL,
"The Chain sub entity authorization Ranger admin web URIs"))
.put(
CHAIN_RANGER_AUTH_TYPES,
PropertyEntry.wildcardPropertyEntry(
CHAIN_RANGER_AUTH_TYPES,
"The Chain sub entity authorization Ranger admin web auth type (kerberos/simple)"))
.put(
CHAIN_RANGER_USERNAME,
PropertyEntry.wildcardPropertyEntry(
CHAIN_RANGER_USERNAME,
"The Chain sub entity authorization Ranger admin web login username"))
.put(
CHAIN_RANGER_PASSWORD,
PropertyEntry.wildcardPropertyEntry(
CHAIN_RANGER_PASSWORD,
"The Chain sub entity authorization Ranger admin web login password"))
.put(
RANGER_SERVICE_NAME,
PropertyEntry.stringOptionalPropertyEntry(
Expand All @@ -65,4 +206,24 @@ public class AuthorizationPropertiesMeta {
PropertyEntry.stringOptionalPropertyEntry(
RANGER_PASSWORD, "The Ranger admin web login password", true, null, false))
.build();

@Override
protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
return AUTHORIZATION_PROPERTY_ENTRIES;
}

@Override
public String firstNodeName() {
return "authorization";
}

@Override
public String secondNodeName() {
return "chain";
}

@Override
public String wildcardNodeName() {
return "plugins";
}
}
11 changes: 11 additions & 0 deletions core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ default boolean isRequiredProperty(String propertyName) {
&& propertyEntries().get(propertyName).isRequired();
}

/**
* Check if the property is wildcard.
*
* @param propertyName The name of the property.
* @return true if the property is existed and wildcard, false otherwise.
*/
default boolean isWildcardProperty(String propertyName) {
return propertyEntries().containsKey(propertyName)
&& propertyEntries().get(propertyName).isWildcard();
}

/**
* Check if the property is immutable.
*
Expand Down
30 changes: 28 additions & 2 deletions core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public final class PropertyEntry<T> {
private final Function<T, String> encoder;
private final boolean hidden;
private final boolean reserved;
private final boolean wildcard;

/**
* @param name The name of the property
Expand All @@ -64,7 +65,8 @@ private PropertyEntry(
Function<String, T> decoder,
Function<T, String> encoder,
boolean hidden,
boolean reserved) {
boolean reserved,
boolean wildcard) {
Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be null or empty");
Preconditions.checkArgument(
StringUtils.isNotBlank(description), "description cannot be null or empty");
Expand All @@ -87,6 +89,7 @@ private PropertyEntry(
this.encoder = encoder;
this.hidden = hidden;
this.reserved = reserved;
this.wildcard = wildcard;
}

public static class Builder<T> {
Expand All @@ -100,6 +103,7 @@ public static class Builder<T> {
private Function<T, String> encoder;
private boolean hidden;
private boolean reserved;
private boolean wildcard;

public Builder<T> withName(String name) {
this.name = name;
Expand Down Expand Up @@ -151,6 +155,11 @@ public Builder<T> withReserved(boolean reserved) {
return this;
}

public Builder<T> withWildcard(boolean wildcard) {
this.wildcard = wildcard;
return this;
}

public PropertyEntry<T> build() {
return new PropertyEntry<T>(
name,
Expand All @@ -162,7 +171,8 @@ public PropertyEntry<T> build() {
decoder,
encoder,
hidden,
reserved);
reserved,
wildcard);
}
}

Expand Down Expand Up @@ -268,6 +278,22 @@ public static PropertyEntry<Boolean> booleanReservedPropertyEntry(
return booleanPropertyEntry(name, description, false, true, defaultValue, hidden, true);
}

public static PropertyEntry<String> wildcardPropertyEntry(String name, String description) {
return new Builder<String>()
.withName(name)
.withDescription(description)
.withRequired(false)
.withImmutable(false)
.withJavaType(String.class)
.withDefaultValue(null)
.withDecoder(Function.identity())
.withEncoder(Function.identity())
.withHidden(false)
.withReserved(false)
.withWildcard(true)
.build();
}

public static PropertyEntry<Boolean> booleanPropertyEntry(
String name,
String description,
Expand Down
Loading

0 comments on commit 9ef2d81

Please sign in to comment.