From edf36d89e2c1b333d03e9e2c327f739ea0796e09 Mon Sep 17 00:00:00 2001
From: Piotr Findeisen <piotr.findeisen@gmail.com>
Date: Wed, 2 Feb 2022 09:14:27 +0100
Subject: [PATCH 1/2] Cleanup Glue databases left behind by tests

---
 .../metastore/glue/TestHiveGlueMetastore.java | 45 ++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java
index 6fd3f32ebcbd..50c9276b30e1 100644
--- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java
+++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/glue/TestHiveGlueMetastore.java
@@ -15,11 +15,17 @@
 
 import com.amazonaws.services.glue.AWSGlueAsync;
 import com.amazonaws.services.glue.AWSGlueAsyncClientBuilder;
+import com.amazonaws.services.glue.model.Database;
+import com.amazonaws.services.glue.model.DeleteDatabaseRequest;
+import com.amazonaws.services.glue.model.EntityNotFoundException;
+import com.amazonaws.services.glue.model.GetDatabasesRequest;
+import com.amazonaws.services.glue.model.GetDatabasesResult;
 import com.amazonaws.services.glue.model.TableInput;
 import com.amazonaws.services.glue.model.UpdateTableRequest;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import io.airlift.concurrent.BoundedExecutor;
+import io.airlift.log.Logger;
 import io.airlift.slice.Slice;
 import io.trino.plugin.hive.AbstractTestHiveLocal;
 import io.trino.plugin.hive.HiveBasicStatistics;
@@ -81,6 +87,7 @@
 import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
 import static io.trino.plugin.hive.acid.AcidTransaction.NO_ACID_TRANSACTION;
 import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics;
+import static io.trino.plugin.hive.metastore.glue.AwsSdkUtil.getPaginatedResults;
 import static io.trino.plugin.hive.metastore.glue.PartitionFilterBuilder.DECIMAL_TYPE;
 import static io.trino.plugin.hive.metastore.glue.PartitionFilterBuilder.decimalOf;
 import static io.trino.spi.connector.RetryMode.NO_RETRIES;
@@ -92,8 +99,10 @@
 import static io.trino.spi.type.VarcharType.VARCHAR;
 import static io.trino.testing.TestingConnectorSession.SESSION;
 import static java.lang.String.format;
+import static java.lang.System.currentTimeMillis;
 import static java.util.Locale.ENGLISH;
 import static java.util.UUID.randomUUID;
+import static java.util.concurrent.TimeUnit.DAYS;
 import static org.apache.hadoop.hive.common.FileUtils.makePartName;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -109,9 +118,12 @@
 public class TestHiveGlueMetastore
         extends AbstractTestHiveLocal
 {
+    private static final Logger log = Logger.get(TestHiveGlueMetastore.class);
+
     private static final HiveIdentity HIVE_IDENTITY = new HiveIdentity(SESSION);
     private static final String PARTITION_KEY = "part_key_1";
     private static final String PARTITION_KEY2 = "part_key_2";
+    private static final String TEST_DATABASE_NAME_PREFIX = "test_glue";
 
     private static final List<ColumnMetadata> CREATE_TABLE_COLUMNS = ImmutableList.<ColumnMetadata>builder()
             .add(new ColumnMetadata("id", BigintType.BIGINT))
@@ -163,7 +175,7 @@ public class TestHiveGlueMetastore
 
     public TestHiveGlueMetastore()
     {
-        super("test_glue" + randomUUID().toString().toLowerCase(ENGLISH).replace("-", ""));
+        super(TEST_DATABASE_NAME_PREFIX + randomUUID().toString().toLowerCase(ENGLISH).replace("-", ""));
     }
 
     protected AWSGlueAsync getGlueClient()
@@ -207,6 +219,37 @@ protected HiveMetastore createMetastore(File tempDir)
                                 .setHideDeltaLakeTables(true)).get());
     }
 
+    @Test
+    public void cleanupOrphanedDatabases()
+    {
+        long creationTimeMillisThreshold = currentTimeMillis() - DAYS.toMillis(1);
+        List<String> orphanedDatabases = getPaginatedResults(
+                glueClient::getDatabases,
+                new GetDatabasesRequest(),
+                GetDatabasesRequest::setNextToken,
+                GetDatabasesResult::getNextToken)
+                .map(GetDatabasesResult::getDatabaseList)
+                .flatMap(List::stream)
+                .filter(database -> database.getName().startsWith(TEST_DATABASE_NAME_PREFIX) &&
+                        database.getCreateTime().getTime() <= creationTimeMillisThreshold)
+                .map(Database::getName)
+                .collect(toImmutableList());
+
+        log.info("Found %s %s* databases that look orphaned, removing", orphanedDatabases.size(), TEST_DATABASE_NAME_PREFIX);
+        orphanedDatabases.forEach(database -> {
+            try {
+                glueClient.deleteDatabase(new DeleteDatabaseRequest()
+                        .withName(database));
+            }
+            catch (EntityNotFoundException e) {
+                log.info("Database [%s] not found, could be removed by other cleanup process", database);
+            }
+            catch (RuntimeException e) {
+                log.warn(e, "Failed to remove database [%s]", database);
+            }
+        });
+    }
+
     @Override
     public void testRenameTable()
     {

From 1024c7c9e5e8f93ee8fe60c1c0eda19ce85dd9f8 Mon Sep 17 00:00:00 2001
From: Piotr Findeisen <piotr.findeisen@gmail.com>
Date: Wed, 2 Feb 2022 09:25:31 +0100
Subject: [PATCH 2/2] empty