Introduce Polaris Admin Tool (with bootstrap and purge commands)

LICENSE-BINARY-DIST

@@ -280,6 +280,7 @@ commons-io:commons-io
 commons-logging:commons-logging
 commons-net:commons-net
 dev.failsafe:failsafe
+info.picocli:picocli
 io.airlift:aircompressor
 io.grpc:grpc-alts
 io.grpc:grpc-api
@@ -400,6 +401,7 @@ io.quarkus:quarkus-micrometer-registry-prometheus
 io.quarkus:quarkus-mutiny
 io.quarkus:quarkus-netty
 io.quarkus:quarkus-opentelemetry
+io.quarkus:quarkus-picocli
 io.quarkus:quarkus-reactive-routes
 io.quarkus:quarkus-rest
 io.quarkus:quarkus-rest-common

extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java

@@ -20,6 +20,7 @@
 
 import io.smallrye.common.annotation.Identifier;
 import jakarta.annotation.Nonnull;
+import jakarta.annotation.Nullable;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import java.nio.file.Path;
@@ -28,6 +29,7 @@
 import org.apache.polaris.core.PolarisDiagnostics;
 import org.apache.polaris.core.context.RealmContext;
 import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
 import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
 import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
@@ -70,14 +72,15 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics
   protected PolarisMetaStoreSession createMetaStoreSession(
       @Nonnull PolarisEclipseLinkStore store,
       @Nonnull RealmContext realmContext,
+      @Nullable PolarisCredentialsBootstrap credentialsBootstrap,
       @Nonnull PolarisDiagnostics diagnostics) {
     return new PolarisEclipseLinkMetaStoreSessionImpl(
         store,
         storageIntegrationProvider,
         realmContext,
         configurationFile(),
         persistenceUnitName(),
-        secretsGenerator(realmContext),
+        secretsGenerator(realmContext, credentialsBootstrap),
         diagnostics);
   }
 

gradle/projects.main.properties

@@ -25,6 +25,7 @@ polaris-api-management-service=api/management-service
 polaris-service-common=service/common
 polaris-quarkus-service=quarkus/service
 polaris-quarkus-server=quarkus/server
+polaris-quarkus-admin=quarkus/admin
 polaris-eclipselink=extension/persistence/eclipselink
 polaris-jpa-model=extension/persistence/jpa-model
 polaris-tests=integration-tests

polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java

@@ -19,6 +19,7 @@
 package org.apache.polaris.core.persistence;
 
 import jakarta.annotation.Nonnull;
+import jakarta.annotation.Nullable;
 import java.time.Clock;
 import java.util.HashMap;
 import java.util.List;
@@ -58,8 +59,6 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType>
   private final PolarisDiagnostics diagnostics;
   private final Clock clock;
 
-  private boolean bootstrap;
-
   protected LocalPolarisMetaStoreManagerFactory(
       PolarisConfigurationStore configurationStore, PolarisDiagnostics diagnostics, Clock clock) {
     this.configurationStore = configurationStore;
@@ -72,45 +71,46 @@ protected LocalPolarisMetaStoreManagerFactory(
   protected abstract PolarisMetaStoreSession createMetaStoreSession(
       @Nonnull StoreType store,
       @Nonnull RealmContext realmContext,
+      @Nullable PolarisCredentialsBootstrap credentialsBootstrap,
       @Nonnull PolarisDiagnostics diagnostics);
 
-  protected PrincipalSecretsGenerator secretsGenerator(RealmContext realmContext) {
-    if (bootstrap) {
-      return PrincipalSecretsGenerator.bootstrap(realmContext.getRealmIdentifier());
+  protected PrincipalSecretsGenerator secretsGenerator(
+      RealmContext realmContext, @Nullable PolarisCredentialsBootstrap credentialsBootstrap) {
+    if (credentialsBootstrap != null) {
+      return PrincipalSecretsGenerator.bootstrap(
+          realmContext.getRealmIdentifier(), credentialsBootstrap);
     } else {
       return PrincipalSecretsGenerator.RANDOM_SECRETS;
     }
   }
 
-  private void initializeForRealm(RealmContext realmContext) {
+  private void initializeForRealm(
+      RealmContext realmContext, PolarisCredentialsBootstrap credentialsBootstrap) {
     final StoreType backingStore = createBackingStore(diagnostics);
     sessionSupplierMap.put(
         realmContext.getRealmIdentifier(),
-        () -> createMetaStoreSession(backingStore, realmContext, diagnostics));
+        () ->
+            createMetaStoreSession(backingStore, realmContext, credentialsBootstrap, diagnostics));
 
     PolarisMetaStoreManager metaStoreManager =
         new PolarisMetaStoreManagerImpl(realmContext, diagnostics, configurationStore, clock);
     metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager);
   }
 
   @Override
-  public synchronized Map<String, PrincipalSecretsResult> bootstrapRealms(List<String> realms) {
+  public synchronized Map<String, PrincipalSecretsResult> bootstrapRealms(
+      List<String> realms, PolarisCredentialsBootstrap credentialsBootstrap) {
     Map<String, PrincipalSecretsResult> results = new HashMap<>();
 
-    bootstrap = true;
-    try {
-      for (String realm : realms) {
-        RealmContext realmContext = () -> realm;
-        if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
-          initializeForRealm(realmContext);
-          PrincipalSecretsResult secretsResult =
-              bootstrapServiceAndCreatePolarisPrincipalForRealm(
-                  realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
-          results.put(realmContext.getRealmIdentifier(), secretsResult);
-        }
+    for (String realm : realms) {
+      RealmContext realmContext = () -> realm;
+      if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
+        initializeForRealm(realmContext, credentialsBootstrap);
+        PrincipalSecretsResult secretsResult =
+            bootstrapServiceAndCreatePolarisPrincipalForRealm(
+                realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
+        results.put(realmContext.getRealmIdentifier(), secretsResult);
       }
-    } finally {
-      bootstrap = false;
     }
 
     return results;
@@ -134,7 +134,7 @@ public void purgeRealms(List<String> realms) {
   public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager(
       RealmContext realmContext) {
     if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
-      initializeForRealm(realmContext);
+      initializeForRealm(realmContext, null);
       checkPolarisServiceBootstrappedForRealm(
           realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
     }
@@ -145,7 +145,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager(
   public synchronized Supplier<PolarisMetaStoreSession> getOrCreateSessionSupplier(
       RealmContext realmContext) {
     if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) {
-      initializeForRealm(realmContext);
+      initializeForRealm(realmContext, null);
       checkPolarisServiceBootstrappedForRealm(
           realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
     } else {

polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java

@@ -37,7 +37,8 @@ public interface MetaStoreManagerFactory {
 
   EntityCache getOrCreateEntityCache(RealmContext realmContext);
 
-  Map<String, PrincipalSecretsResult> bootstrapRealms(List<String> realms);
+  Map<String, PrincipalSecretsResult> bootstrapRealms(
+      List<String> realms, PolarisCredentialsBootstrap credentialsBootstrap);
 
   /** Purge all metadata for the realms provided */
   void purgeRealms(List<String> realms);

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrap.java

@@ -35,6 +35,9 @@
  */
 public class PolarisCredentialsBootstrap {
 
+  public static final PolarisCredentialsBootstrap EMPTY =
+      new PolarisCredentialsBootstrap(new HashMap<>());
+
   /**
    * Parse credentials from the system property {@code polaris.bootstrap.credentials} or the
    * environment variable {@code POLARIS_BOOTSTRAP_CREDENTIALS}, whichever is set.
@@ -55,35 +58,38 @@ public static PolarisCredentialsBootstrap fromEnvironment() {
    * </pre>
    */
   public static PolarisCredentialsBootstrap fromString(@Nullable String credentialsString) {
+    return credentialsString != null && !credentialsString.isBlank()
+        ? fromList(Splitter.on(';').trimResults().splitToList(credentialsString))
+        : EMPTY;
+  }
+
+  /**
+   * Parse a list of credentials; each element should be in the format: {@code
+   * realm,principal,clientId,clientSecret}.
+   */
+  public static PolarisCredentialsBootstrap fromList(List<String> credentialsList) {
     Map<String, Map<String, Map.Entry<String, String>>> credentials = new HashMap<>();
-    if (credentialsString != null && !credentialsString.isBlank()) {
-      Splitter.on(';')
-          .trimResults()
-          .splitToList(credentialsString)
-          .forEach(
-              quadruple -> {
-                if (!quadruple.isBlank()) {
-                  List<String> parts = Splitter.on(',').trimResults().splitToList(quadruple);
-                  if (parts.size() != 4) {
-                    throw new IllegalArgumentException("Invalid credentials format: " + quadruple);
-                  }
-                  String realmName = parts.get(0);
-                  String principalName = parts.get(1);
-                  String clientId = parts.get(2);
-                  String clientSecret = parts.get(3);
-                  credentials
-                      .computeIfAbsent(realmName, k -> new HashMap<>())
-                      .merge(
-                          principalName,
-                          new SimpleEntry<>(clientId, clientSecret),
-                          (a, b) -> {
-                            throw new IllegalArgumentException(
-                                "Duplicate principal: " + principalName);
-                          });
-                }
-              });
+    for (String quadruple : credentialsList) {
+      if (!quadruple.isBlank()) {
+        List<String> parts = Splitter.on(',').trimResults().splitToList(quadruple);
+        if (parts.size() != 4) {
+          throw new IllegalArgumentException("Invalid credentials format: " + quadruple);
+        }
+        String realmName = parts.get(0);
+        String principalName = parts.get(1);
+        String clientId = parts.get(2);
+        String clientSecret = parts.get(3);
+        credentials
+            .computeIfAbsent(realmName, k -> new HashMap<>())
+            .merge(
+                principalName,
+                new SimpleEntry<>(clientId, clientSecret),
+                (a, b) -> {
+                  throw new IllegalArgumentException("Duplicate principal: " + principalName);
+                });
+      }
     }
-    return new PolarisCredentialsBootstrap(credentials);
+    return credentials.isEmpty() ? EMPTY : new PolarisCredentialsBootstrap(credentials);
   }
 
   @VisibleForTesting final Map<String, Map<String, Map.Entry<String, String>>> credentials;

polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrapTest.java

@@ -22,6 +22,7 @@
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import java.util.Comparator;
+import java.util.List;
 import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
 import org.junit.jupiter.api.Test;
 
@@ -87,6 +88,27 @@ void getSecretsValidString() {
         .contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a"));
   }
 
+  @Test
+  void getSecretsValidList() {
+    PolarisCredentialsBootstrap credentials =
+        PolarisCredentialsBootstrap.fromList(
+            List.of(
+                "realm1,user1a,client1a,secret1a",
+                "realm1,user1b,client1b,secret1b",
+                "realm2,user2a,client2a,secret2a"));
+    assertThat(credentials.getSecrets("realm1", 123, "nonexistent")).isEmpty();
+    assertThat(credentials.getSecrets("nonexistent", 123, "user1a")).isEmpty();
+    assertThat(credentials.getSecrets("realm1", 123, "user1a"))
+        .usingValueComparator(comparator)
+        .contains(new PolarisPrincipalSecrets(123, "client1a", "secret1a", "secret1a"));
+    assertThat(credentials.getSecrets("realm1", 123, "user1b"))
+        .usingValueComparator(comparator)
+        .contains(new PolarisPrincipalSecrets(123, "client1b", "secret1b", "secret1b"));
+    assertThat(credentials.getSecrets("realm2", 123, "user2a"))
+        .usingValueComparator(comparator)
+        .contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a"));
+  }
+
   @Test
   void getSecretsValidSystemProperty() {
     PolarisCredentialsBootstrap credentials = PolarisCredentialsBootstrap.fromEnvironment();

quarkus/admin/README.md

@@ -0,0 +1,27 @@
+# Polaris Admin Tool
+
+This module contains a maintenance tool for performing administrative tasks on the Polaris database.
+It is a Quarkus application that can be used to perform various maintenance tasks targeting the
+Polaris database directly.
+
+Building this module will create a runnable uber-jar that can be executed from the command line.
+
+To also build the Docker image, you can use the following command:
+
+```shell
+./gradlew :polaris-quarkus-admin:assemble -Dquarkus.container-image.build=true
+```
+
+## Running the Admin Tool
+
+The admin tool can be run from the command line using the following command:
+
+```shell
+java -jar polaris-quarkus-admin-<version>-runner.jar --help
+```
+
+Using the Docker image, you can run the admin tool with the following command:
+
+```shell
+docker run --rm -it polaris-admin-tool:<version> --help
+```

quarkus/admin/build.gradle.kts

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import io.quarkus.gradle.tasks.QuarkusBuild
+
+plugins {
+  alias(libs.plugins.quarkus)
+  alias(libs.plugins.openapi.generator)
+  id("polaris-server")
+  id("polaris-license-report")
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+  implementation(project(":polaris-api-management-service"))
+  implementation(project(":polaris-api-iceberg-service"))
+  implementation(project(":polaris-service-common"))
+  implementation(project(":polaris-quarkus-service"))
+
+  implementation(enforcedPlatform(libs.quarkus.bom))
+  implementation("io.quarkus:quarkus-picocli")
+  implementation("io.quarkus:quarkus-container-image-docker")
+
+  implementation("org.jboss.slf4j:slf4j-jboss-logmanager")
+
+  // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329
+  implementation(platform(libs.dnsjava))
+
+  testImplementation(enforcedPlatform(libs.quarkus.bom))
+  testImplementation("io.quarkus:quarkus-junit5")
+}
+
+tasks.withType<ProcessResources>().configureEach {
+  from("src/main/resources") {
+    expand("polarisVersion" to version)
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+  }
+}
+
+quarkus {
+  quarkusBuildProperties.put("quarkus.package.type", "uber-jar")
+  // Pull manifest attributes from the "main" `jar` task to get the
+  // release-information into the jars generated by Quarkus.
+  quarkusBuildProperties.putAll(
+    provider {
+      tasks
+        .named("jar", Jar::class.java)
+        .get()
+        .manifest
+        .attributes
+        .map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() }
+        .toMap()
+    }
+  )
+}
+
+publishing {
+  publications {
+    named<MavenPublication>("maven") {
+      val quarkusBuild = tasks.getByName<QuarkusBuild>("quarkusBuild")
+      artifact(quarkusBuild.runnerJar) {
+        classifier = "runner"
+        builtBy(quarkusBuild)
+      }
+    }
+  }
+}